From b5fabf412cf55b83a0ceda7991ace83b983ad0a0 Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 18:14:19 +0530 Subject: [PATCH 1/7] Add TrigramSearchManager to ChaloBEST models. --- chaloBEST/mumbai/models.py | 21 +++++++++++++++++++++ gateway/settings.py | 3 +-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/chaloBEST/mumbai/models.py b/chaloBEST/mumbai/models.py index dcae8e9..e5e81a4 100644 --- a/chaloBEST/mumbai/models.py +++ b/chaloBEST/mumbai/models.py @@ -42,7 +42,27 @@ SCHED = { '2nd &4th':['???'] } +class TrigramSearchManager(models.Manager): + def set_threshold(self, threshold): + """Set the limit for trigram similarity matching.""" + cursor = connection.cursor() + cursor.execute("""SELECT set_limit(%f)""" % threshold) + + def find_approximate(self, match=0.5, **kwargs): + self.set_threshold(match) + assert(len(kwargs) == 1) + column, value = kwargs.items()[0] + qset = self.get_query_set() + # use the pg_trgm index via the % operator + qset = qset.extra(select={"similarity":"similarity(" + column + ", %s)"}, + select_params=[value], + where=[column + " %% %s"], + params=[value], + order_by=["-similarity"]) + return qset + class Area(models.Model): + objects = TrigramSearchManager() # name, name_mr code = models.IntegerField() #primary_key=True) slug = models.SlugField(null=True) name = models.TextField(blank=True, max_length=255) @@ -93,6 +113,7 @@ class Fare(models.Model): class Stop(models.Model): + objects = TrigramSearchManager() # name, display, name_mr code = models.IntegerField() slug = models.SlugField(null=True) name = models.TextField(blank=True, max_length=255) diff --git a/gateway/settings.py b/gateway/settings.py index 5cef581..ce79173 100755 --- a/gateway/settings.py +++ b/gateway/settings.py @@ -2,8 +2,6 @@ # vim: ai ts=4 sts=4 et sw=4 # encoding=utf-8 -# Put this in /srv/smsBEST/gateway and change the gateway secret. - # -------------------------------------------------------------------- # # MAIN CONFIGURATION # # -------------------------------------------------------------------- # @@ -51,6 +49,7 @@ GATEWAY = { "push": "http://chalobest.in:8086/?from=%(from)s&txt=%(txt)s&secret=%(secret)s" } +AJAX_PROXY_HOST = "0.0.0.0" # to open the gateway from the outside # to help you get started quickly, many django/rapidsms apps are enabled # by default. you may wish to remove some and/or add your own. From 0cf8fae70bf4aa36843d90764ad8fb341006e954 Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 18:55:35 +0530 Subject: [PATCH 2/7] Somewhat re-thought TrigramSearchManager. --- chaloBEST/mumbai/models.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/chaloBEST/mumbai/models.py b/chaloBEST/mumbai/models.py index e5e81a4..da00dc0 100644 --- a/chaloBEST/mumbai/models.py +++ b/chaloBEST/mumbai/models.py @@ -43,26 +43,32 @@ SCHED = { } class TrigramSearchManager(models.Manager): + def __init__(self, trigram_columns, *args, **kwargs): + super(models.Manager, self).__init__(*args, **kwargs) + self.trigram_columns = trigram_columns + def set_threshold(self, threshold): """Set the limit for trigram similarity matching.""" cursor = connection.cursor() cursor.execute("""SELECT set_limit(%f)""" % threshold) - def find_approximate(self, match=0.5, **kwargs): + def find_approximate(self, text, match=0.5): self.set_threshold(match) - assert(len(kwargs) == 1) - column, value = kwargs.items()[0] + similarity_measure = "max(%s)" % ",".join(["similarity(%s, %%s)" % col for col in self.trigram_columns]) + similarity_filter = " OR ".join(["%s %%%% %%s" % col for col in self.trigram_columns]) + text_values = [text] * len(self.trigram_columns) + qset = self.get_query_set() # use the pg_trgm index via the % operator - qset = qset.extra(select={"similarity":"similarity(" + column + ", %s)"}, - select_params=[value], - where=[column + " %% %s"], - params=[value], + qset = qset.extra(select={"similarity":similarity_measure, + select_params=text_values, + where=similarity_filter, + params=text_values, order_by=["-similarity"]) return qset class Area(models.Model): - objects = TrigramSearchManager() # name, name_mr + objects = TrigramSearchManager("name", "name_mr", "display_name") code = models.IntegerField() #primary_key=True) slug = models.SlugField(null=True) name = models.TextField(blank=True, max_length=255) @@ -111,9 +117,8 @@ class Fare(models.Model): def __unicode__(self): return str(self.slab) - class Stop(models.Model): - objects = TrigramSearchManager() # name, display, name_mr + objects = TrigramSearchManager("name", "name_mr", "display_name") code = models.IntegerField() slug = models.SlugField(null=True) name = models.TextField(blank=True, max_length=255) From ddfbe2646ece231cfce0a35d751433d998aa691f Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 19:00:24 +0530 Subject: [PATCH 3/7] Somewhat re-thought TrigramSearchManager (using greatest instead of max). --- chaloBEST/mumbai/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chaloBEST/mumbai/models.py b/chaloBEST/mumbai/models.py index da00dc0..414effc 100644 --- a/chaloBEST/mumbai/models.py +++ b/chaloBEST/mumbai/models.py @@ -54,7 +54,7 @@ class TrigramSearchManager(models.Manager): def find_approximate(self, text, match=0.5): self.set_threshold(match) - similarity_measure = "max(%s)" % ",".join(["similarity(%s, %%s)" % col for col in self.trigram_columns]) + similarity_measure = "greatest(%s)" % ",".join(["similarity(%s, %%s)" % col for col in self.trigram_columns]) similarity_filter = " OR ".join(["%s %%%% %%s" % col for col in self.trigram_columns]) text_values = [text] * len(self.trigram_columns) From e8e5b6e85a7bb6a58b25865b8b0632238c774a8f Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 19:14:47 +0530 Subject: [PATCH 4/7] Add mgmt command to create trigram indexes. --- chaloBEST/mumbai/management/__init__.py | 0 .../mumbai/management/commands/__init__.py | 0 .../mumbai/management/commands/trgmidx.py | 18 ++++++++++++++++++ 3 files changed, 18 insertions(+) create mode 100644 chaloBEST/mumbai/management/__init__.py create mode 100644 chaloBEST/mumbai/management/commands/__init__.py create mode 100644 chaloBEST/mumbai/management/commands/trgmidx.py diff --git a/chaloBEST/mumbai/management/__init__.py b/chaloBEST/mumbai/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chaloBEST/mumbai/management/commands/__init__.py b/chaloBEST/mumbai/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chaloBEST/mumbai/management/commands/trgmidx.py b/chaloBEST/mumbai/management/commands/trgmidx.py new file mode 100644 index 0000000..000e0d7 --- /dev/null +++ b/chaloBEST/mumbai/management/commands/trgmidx.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db import connection +from mumbai import models + +class Command(BaseCommand): + help = "Instantiates the pg_trgm indexes" + + def handle(self, *args, **options): + cursor = connection.cursor() + for name, model in models: + if not hasattr(model, "objects") or \ + not isinstance(model.objects, model.TrigramSearchManager): + continue + table = model._meta.db_table + for column in model.objects.trigram_columns: + cursor.execute(""" + CREATE INDEX %s_%s_trgm_idx ON %s USING gin (%s gin_trgm_ops);""" % ( + table, column, table, column)) From 44e91fe2cafa3b611964c2490d1d935dfab9709b Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 07:00:28 -0800 Subject: [PATCH 5/7] Fix the dummheiten in the TrigramSearchManager. --- chaloBEST/mumbai/management/commands/trgmidx.py | 12 +++++++----- chaloBEST/mumbai/models.py | 14 +++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/chaloBEST/mumbai/management/commands/trgmidx.py b/chaloBEST/mumbai/management/commands/trgmidx.py index 000e0d7..b4af2d3 100644 --- a/chaloBEST/mumbai/management/commands/trgmidx.py +++ b/chaloBEST/mumbai/management/commands/trgmidx.py @@ -7,12 +7,14 @@ class Command(BaseCommand): def handle(self, *args, **options): cursor = connection.cursor() - for name, model in models: + for name in dir(models): + model = getattr(models, name) if not hasattr(model, "objects") or \ - not isinstance(model.objects, model.TrigramSearchManager): + not isinstance(model.objects, models.TrigramSearchManager): continue table = model._meta.db_table for column in model.objects.trigram_columns: - cursor.execute(""" - CREATE INDEX %s_%s_trgm_idx ON %s USING gin (%s gin_trgm_ops);""" % ( - table, column, table, column)) + sql = """CREATE INDEX %s_%s_trgm_idx ON %s USING gin (%s gin_trgm_ops);""" % ( + table, column, table, column) + cursor.execute(sql) + cursor.execute("COMMIT;") diff --git a/chaloBEST/mumbai/models.py b/chaloBEST/mumbai/models.py index 414effc..bae603d 100644 --- a/chaloBEST/mumbai/models.py +++ b/chaloBEST/mumbai/models.py @@ -3,6 +3,7 @@ from django.contrib.gis.geos import Point from django import forms from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import generic +from django.db import connection import json STOP_CHOICES = ( ('U','Up'), @@ -43,8 +44,8 @@ SCHED = { } class TrigramSearchManager(models.Manager): - def __init__(self, trigram_columns, *args, **kwargs): - super(models.Manager, self).__init__(*args, **kwargs) + def __init__(self, trigram_columns=[]): + super(TrigramSearchManager, self).__init__() self.trigram_columns = trigram_columns def set_threshold(self, threshold): @@ -57,18 +58,17 @@ class TrigramSearchManager(models.Manager): similarity_measure = "greatest(%s)" % ",".join(["similarity(%s, %%s)" % col for col in self.trigram_columns]) similarity_filter = " OR ".join(["%s %%%% %%s" % col for col in self.trigram_columns]) text_values = [text] * len(self.trigram_columns) - qset = self.get_query_set() # use the pg_trgm index via the % operator - qset = qset.extra(select={"similarity":similarity_measure, + qset = qset.extra(select={"similarity":similarity_measure}, select_params=text_values, - where=similarity_filter, + where=[similarity_filter], params=text_values, order_by=["-similarity"]) return qset class Area(models.Model): - objects = TrigramSearchManager("name", "name_mr", "display_name") + objects = TrigramSearchManager(("name", "name_mr", "display_name")) code = models.IntegerField() #primary_key=True) slug = models.SlugField(null=True) name = models.TextField(blank=True, max_length=255) @@ -118,7 +118,7 @@ class Fare(models.Model): return str(self.slab) class Stop(models.Model): - objects = TrigramSearchManager("name", "name_mr", "display_name") + objects = TrigramSearchManager(("name", "name_mr", "display_name")) code = models.IntegerField() slug = models.SlugField(null=True) name = models.TextField(blank=True, max_length=255) From 655e95d7cd2e2bdbcec44e6138e0ce6ebae05b28 Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 08:59:30 -0800 Subject: [PATCH 6/7] Add RapidSMS requirement; create smsBEST RapidSMS project. --- requirements.txt | 2 + smsBEST/__init__.py | 0 smsBEST/manage.py | 18 ++++ smsBEST/mumbai/__init__.py | 0 smsBEST/mumbai/app.py | 4 + smsBEST/mumbai/views.py | 1 + smsBEST/settings.py | 205 +++++++++++++++++++++++++++++++++++++ smsBEST/urls.py | 37 +++++++ 8 files changed, 267 insertions(+) create mode 100644 smsBEST/__init__.py create mode 100755 smsBEST/manage.py create mode 100644 smsBEST/mumbai/__init__.py create mode 100644 smsBEST/mumbai/app.py create mode 100644 smsBEST/mumbai/views.py create mode 100644 smsBEST/settings.py create mode 100644 smsBEST/urls.py diff --git a/requirements.txt b/requirements.txt index a239605..226e914 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ -e git+git://github.com/bit/django-extensions.git#egg=django_extensions #django_extensions django-grappelli +-e git+git://github.com/schuyler/rapidsms.git#egg=rapidsms +-e git+git://github.com/schuyler/arrest.git#egg=arrest diff --git a/smsBEST/__init__.py b/smsBEST/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smsBEST/manage.py b/smsBEST/manage.py new file mode 100755 index 0000000..b23e297 --- /dev/null +++ b/smsBEST/manage.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 + +#import sys, os + +from django.core.management import execute_manager +import settings + + +if __name__ == "__main__": +# project_root = os.path.abspath( +# os.path.dirname(__file__)) + +# path = os.path.join(project_root, "apps") +# sys.path.insert(0, path) + +# sys.path.insert(0, project_root) + execute_manager(settings) diff --git a/smsBEST/mumbai/__init__.py b/smsBEST/mumbai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smsBEST/mumbai/app.py b/smsBEST/mumbai/app.py new file mode 100644 index 0000000..e87ddb0 --- /dev/null +++ b/smsBEST/mumbai/app.py @@ -0,0 +1,4 @@ +from rapidsms.apps.base import AppBase + +class App(AppBase): + pass diff --git a/smsBEST/mumbai/views.py b/smsBEST/mumbai/views.py new file mode 100644 index 0000000..60f00ef --- /dev/null +++ b/smsBEST/mumbai/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/smsBEST/settings.py b/smsBEST/settings.py new file mode 100644 index 0000000..fc14793 --- /dev/null +++ b/smsBEST/settings.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python +# vim: ai ts=4 sts=4 et sw=4 +# encoding=utf-8 + +# -------------------------------------------------------------------- # +# MAIN CONFIGURATION # +# -------------------------------------------------------------------- # + + +# you should configure your database here before doing any real work. +# see: http://docs.djangoproject.com/en/dev/ref/settings/#databases +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": "smsbest", + } +} + + +# the rapidsms backend configuration is designed to resemble django's +# database configuration, as a nested dict of (name, configuration). +# +# the ENGINE option specifies the module of the backend; the most common +# backend types (for a GSM modem or an SMPP server) are bundled with +# rapidsms, but you may choose to write your own. +# +# all other options are passed to the Backend when it is instantiated, +# to configure it. see the documentation in those modules for a list of +# the valid options for each. +INSTALLED_BACKENDS = { + #"att": { + # "ENGINE": "rapidsms.backends.gsm", + # "PORT": "/dev/ttyUSB0" + #}, + #"verizon": { + # "ENGINE": "rapidsms.backends.gsm, + # "PORT": "/dev/ttyUSB1" + #}, + "atlas": { + "ENGINE": "rapidsms.backends.http", + "host": "0.0.0.0", + "port": 8086, + "gateway_url": "http://atlas.gnowledge.org:8001/gateway/send", + "params_outgoing": "secret=something+secret&to=%(phone_number)s&txt=%(message)s", + "params_incoming": "from=%(phone_number)s&txt=%(message)s" + + }, + "message_tester": { + "ENGINE": "rapidsms.backends.bucket", + } +} + + +# to help you get started quickly, many django/rapidsms apps are enabled +# by default. you may wish to remove some and/or add your own. +INSTALLED_APPS = [ + + # the essentials. + "django_nose", + "djtables", + "rapidsms", + + # common dependencies (which don't clutter up the ui). + "rapidsms.contrib.handlers", + "rapidsms.contrib.ajax", + + # enable the django admin using a little shim app (which includes + # the required urlpatterns), and a bunch of undocumented apps that + # the AdminSite seems to explode without. + "django.contrib.sites", + "django.contrib.auth", + "django.contrib.admin", + "django.contrib.sessions", + "django.contrib.contenttypes", + + # the rapidsms contrib apps. + # "rapidsms.contrib.default", + # "rapidsms.contrib.export", + "rapidsms.contrib.httptester", + # "rapidsms.contrib.locations", + "rapidsms.contrib.messagelog", + "rapidsms.contrib.messaging", + "rapidsms.contrib.registration", + # "rapidsms.contrib.scheduler", + # "rapidsms.contrib.echo", + + "mumbai" +] + + + +# this rapidsms-specific setting defines which views are linked by the +# tabbed navigation. when adding an app to INSTALLED_APPS, you may wish +# to add it here, also, to expose it in the rapidsms ui. +RAPIDSMS_TABS = [ + ("rapidsms.contrib.messagelog.views.message_log", "Message Log"), + ("rapidsms.contrib.registration.views.registration", "Registration"), + ("rapidsms.contrib.messaging.views.messaging", "Messaging"), + # ("rapidsms.contrib.locations.views.locations", "Map"), + # ("rapidsms.contrib.scheduler.views.index", "Event Scheduler"), + ("rapidsms.contrib.httptester.views.generate_identity", "Message Tester"), +] + + +# -------------------------------------------------------------------- # +# BORING CONFIGURATION # +# -------------------------------------------------------------------- # + + +# debug mode is turned on as default, since rapidsms is under heavy +# development at the moment, and full stack traces are very useful +# when reporting bugs. don't forget to turn this off in production. +DEBUG = TEMPLATE_DEBUG = True + + +# after login (which is handled by django.contrib.auth), redirect to the +# dashboard rather than 'accounts/profile' (the default). +LOGIN_REDIRECT_URL = "/" + + +# use django-nose to run tests. rapidsms contains lots of packages and +# modules which django does not find automatically, and importing them +# all manually is tiresome and error-prone. +TEST_RUNNER = "django_nose.NoseTestSuiteRunner" + + +# for some reason this setting is blank in django's global_settings.py, +# but it is needed for static assets to be linkable. +MEDIA_URL = "/static/" + + +# this is required for the django.contrib.sites tests to run, but also +# not included in global_settings.py, and is almost always ``1``. +# see: http://docs.djangoproject.com/en/dev/ref/contrib/sites/ +SITE_ID = 1 + + +# the default log settings are very noisy. +LOG_LEVEL = "DEBUG" +LOG_FILE = "rapidsms.log" +LOG_FORMAT = "[%(name)s]: %(message)s" +LOG_SIZE = 8192 # 8192 bits = 8 kb +LOG_BACKUPS = 256 # number of logs to keep + + +# these weird dependencies should be handled by their respective apps, +# but they're not, so here they are. most of them are for django admin. +TEMPLATE_CONTEXT_PROCESSORS = [ + "django.core.context_processors.auth", + "django.core.context_processors.debug", + "django.core.context_processors.i18n", + "django.core.context_processors.media", + "django.core.context_processors.request", +] + +# template loaders load templates from various places. +# for djtables to work properly, the egg loader needs to be +# included, the others are fairly standard. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + 'django.template.loaders.eggs.Loader' +) + +# -------------------------------------------------------------------- # +# HERE BE DRAGONS! # +# these settings are pure hackery, and will go away soon # +# -------------------------------------------------------------------- # + + +# these apps should not be started by rapidsms in your tests, however, +# the models and bootstrap will still be available through django. +TEST_EXCLUDED_APPS = [ + "django.contrib.sessions", + "django.contrib.contenttypes", + "django.contrib.auth", + "rapidsms", + "rapidsms.contrib.ajax", + "rapidsms.contrib.httptester", +] + +# the project-level url patterns +ROOT_URLCONF = "urls" + +# import local_settings.py +try: + from local_settings import * +except: + pass + +# since we might hit the database from any thread during testing, the +# in-memory sqlite database isn't sufficient. it spawns a separate +# virtual database for each thread, and syncdb is only called for the +# first. this leads to confusing "no such table" errors. We create +# a named temporary instance instead. +import os +import tempfile +import sys + +if 'test' in sys.argv: + for db_name in DATABASES: + DATABASES[db_name]['TEST_NAME'] = os.path.join( + tempfile.gettempdir(), + "%s.rapidsms.test.sqlite3" % db_name) + diff --git a/smsBEST/urls.py b/smsBEST/urls.py new file mode 100644 index 0000000..34efec2 --- /dev/null +++ b/smsBEST/urls.py @@ -0,0 +1,37 @@ +from django.conf.urls.defaults import * +from django.conf import settings +from django.contrib import admin + +admin.autodiscover() + +urlpatterns = patterns('', + # Example: + # (r'^my-project/', include('my_project.foo.urls')), + + # Uncomment the admin/doc line below to enable admin documentation: + # (r'^admin/doc/', include('django.contrib.admindocs.urls')), + + (r'^admin/', include(admin.site.urls)), + + # RapidSMS core URLs + (r'^account/', include('rapidsms.urls.login_logout')), + url(r'^$', 'rapidsms.views.dashboard', name='rapidsms-dashboard'), + + # RapidSMS contrib app URLs + (r'^ajax/', include('rapidsms.contrib.ajax.urls')), + (r'^export/', include('rapidsms.contrib.export.urls')), + (r'^httptester/', include('rapidsms.contrib.httptester.urls')), + (r'^locations/', include('rapidsms.contrib.locations.urls')), + (r'^messagelog/', include('rapidsms.contrib.messagelog.urls')), + (r'^messaging/', include('rapidsms.contrib.messaging.urls')), + (r'^registration/', include('rapidsms.contrib.registration.urls')), + (r'^scheduler/', include('rapidsms.contrib.scheduler.urls')), +) + +if settings.DEBUG: + urlpatterns += patterns('', + # helper URLs file that automatically serves the 'static' folder in + # INSTALLED_APPS via the Django static media server (NOT for use in + # production) + (r'^', include('rapidsms.urls.static_media')), + ) From d04f61ab78b70d969f68f7fd49a4e51ba8af0ae8 Mon Sep 17 00:00:00 2001 From: Schuyler Erle Date: Tue, 28 Feb 2012 10:10:22 -0800 Subject: [PATCH 7/7] More or less working mumbai.app --- smsBEST/mumbai/app.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/smsBEST/mumbai/app.py b/smsBEST/mumbai/app.py index e87ddb0..1e566a0 100644 --- a/smsBEST/mumbai/app.py +++ b/smsBEST/mumbai/app.py @@ -1,4 +1,21 @@ from rapidsms.apps.base import AppBase +import re +import arrest + +DIGIT = re.compile(r"\d{1,3}") +chalobest = arrest.Client("http://chalobest.in/1.0") class App(AppBase): - pass + def handle(self, msg): + if DIGIT.search(msg.text): + routes = chalobest.routes(q=msg.text.replace(" ", "")) + detail = chalobest.route[routes[0]] + stops = detail['stops']['features'] + origin, destination = stops[0]['properties'], stops[-1]['properties'] + msg.respond("%s: %s (%s) to %s (%s)" % (routes[0], + origin['display_name'], origin['area'], + destination['display_name'], destination['area'])) + else: + stops = chalobest.stops(q=msg.text) + stop = stops['features'][0]['properties'] + msg.respond("%s (%s): %s" % (stop['official_name'], stop['area'], stop['routes']))