views call find_approximate for queries

chaloBEST/mumbai/apiviews.py

@@ -29,28 +29,31 @@ def area(request, slug):
     })
 
 def routes(request):
-    qset = Route.objects.all()
+    q = request.GET.get("q", "")
-    if request.GET.has_key('q'):
+    if q != '':
-        q = request.GET.get('q', '')
+        qset = Route.objects.find_approximate(q, 0.33)
-        qset = qset.filter(alias__icontains=q) #FIXME: make a better Q object
+    else:
+        qset = Route.objects.all()
     routes = [route.alias for route in qset]
     return render_to_json_response(routes)
 
 
 def areas(request):
-    qset = Area.objects.all()
+    q = request.GET.get("q", "")
-    if request.GET.has_key('q'):
+    if q != '':
-        q = request.GET.get('q', '')
+        qset = Area.objects.find_approximate(q, 0.33)
-        qset = qset.filter(display_name__icontains=q)    
+    else:
+        qset = Area.objects.all()
     areas = [area.slug for area in qset]
     return render_to_json_response(areas)
 
 def stops(request):
-    qset = Stop.objects.all()
+    q = request.GET.get("q", "")
-    srid = int(request.GET.get("srid", 4326))
+    if q != '':
-    if request.GET.has_key('q'):
+        qset = Stop.objects.find_approximate(q, 0.33)
-        q = request.GET.get('q', '')
+    else:
-        qset = qset.filter(display_name__icontains=q) #FIXME: This definitely needs to be a Q object with OR lookups for area name, road name, etc.    
+        qset = Stop.objects.all()
+    srid = int(request.GET.get("srid", 4326))   
     return render_to_json_response({
         'type': 'FeatureCollection',
         'features': [stop.get_geojson(srid=srid) for stop in qset]

chaloBEST/mumbai/management/commands/trgmidx.py

@@ -0,0 +1,20 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+from mumbai import models
+
+class Command(BaseCommand):
+    help = "Instantiates the pg_trgm indexes"
+
+    def handle(self, *args, **options):
+        cursor = connection.cursor()
+        for name in dir(models):
+            model = getattr(models, name)
+            if not hasattr(model, "objects") or \
+               not isinstance(model.objects, models.TrigramSearchManager):
+                continue
+            table = model._meta.db_table
+            for column in model.objects.trigram_columns:
+                sql = """CREATE INDEX %s_%s_trgm_idx ON %s USING gin (%s gin_trgm_ops);""" % (
+                        table, column, table, column)
+                cursor.execute(sql)
+        cursor.execute("COMMIT;")

chaloBEST/mumbai/models.py

@@ -3,6 +3,7 @@ from django.contrib.gis.geos import Point
 from django import forms
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes import generic
+from django.db import connection
 import json
 
 STOP_CHOICES = ( ('U','Up'),
@@ -42,7 +43,32 @@ SCHED = {
     '2nd &4th':['???']
     }
 
+class TrigramSearchManager(models.Manager):
+    def __init__(self, trigram_columns=[]):
+        super(TrigramSearchManager, self).__init__()
+        self.trigram_columns = trigram_columns
+
+    def set_threshold(self, threshold):
+        """Set the limit for trigram similarity matching."""
+        cursor = connection.cursor()
+        cursor.execute("""SELECT set_limit(%f)""" % threshold)
+
+    def find_approximate(self, text, match=0.5):
+        self.set_threshold(match)
+        similarity_measure = "greatest(%s)" % ",".join(["similarity(%s, %%s)" % col for col in self.trigram_columns])
+        similarity_filter = " OR ".join(["%s %%%% %%s" % col for col in self.trigram_columns])
+        text_values = [text] * len(self.trigram_columns)
+        qset = self.get_query_set()
+        # use the pg_trgm index via the % operator
+        qset = qset.extra(select={"similarity":similarity_measure},
+                          select_params=text_values,
+                          where=[similarity_filter],
+                          params=text_values,
+                          order_by=["-similarity"])
+        return qset
+
 class Area(models.Model):
+    objects = TrigramSearchManager(("name", "name_mr", "display_name"))
     code = models.IntegerField() #primary_key=True)
     slug = models.SlugField(null=True)
     name = models.TextField(blank=True, max_length=255)
@@ -91,8 +117,8 @@ class Fare(models.Model):
     def __unicode__(self):
         return str(self.slab)   
 
-
 class Stop(models.Model):
+    objects = TrigramSearchManager(("name", "name_mr", "display_name"))
     code = models.IntegerField()
     slug = models.SlugField(null=True)
     name = models.TextField(blank=True, max_length=255)

gateway/settings.py

@@ -2,8 +2,6 @@
 # vim: ai ts=4 sts=4 et sw=4
 # encoding=utf-8
 
-# Put this in /srv/smsBEST/gateway and change the gateway secret.
-
 # -------------------------------------------------------------------- #
 #                          MAIN CONFIGURATION                          #
 # -------------------------------------------------------------------- #
@@ -51,6 +49,7 @@ GATEWAY = {
     "push": "http://chalobest.in:8086/?from=%(from)s&txt=%(txt)s&secret=%(secret)s"
 }
 
+AJAX_PROXY_HOST = "0.0.0.0" # to open the gateway from the outside
 
 # to help you get started quickly, many django/rapidsms apps are enabled
 # by default. you may wish to remove some and/or add your own.

requirements.txt

@@ -5,3 +5,5 @@
 -e git+git://github.com/bit/django-extensions.git#egg=django_extensions
 #django_extensions
 django-grappelli
+-e git+git://github.com/schuyler/rapidsms.git#egg=rapidsms
+-e git+git://github.com/schuyler/arrest.git#egg=arrest

smsBEST/manage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# vim: ai ts=4 sts=4 et sw=4
+
+#import sys, os
+
+from django.core.management import execute_manager
+import settings
+
+
+if __name__ == "__main__":
+#    project_root = os.path.abspath(
+#        os.path.dirname(__file__))
+
+#    path = os.path.join(project_root, "apps")
+#    sys.path.insert(0, path)
+
+#    sys.path.insert(0, project_root)
+    execute_manager(settings)

smsBEST/mumbai/app.py

@@ -0,0 +1,21 @@
+from rapidsms.apps.base import AppBase 
+import re
+import arrest
+
+DIGIT = re.compile(r"\d{1,3}")
+chalobest = arrest.Client("http://chalobest.in/1.0")
+
+class App(AppBase):
+    def handle(self, msg):
+        if DIGIT.search(msg.text):
+            routes = chalobest.routes(q=msg.text.replace(" ", ""))
+            detail = chalobest.route[routes[0]]
+            stops = detail['stops']['features']
+            origin, destination = stops[0]['properties'], stops[-1]['properties']
+            msg.respond("%s: %s (%s) to %s (%s)" % (routes[0],
+                    origin['display_name'], origin['area'],
+                    destination['display_name'], destination['area']))
+        else:
+            stops = chalobest.stops(q=msg.text)
+            stop = stops['features'][0]['properties']
+            msg.respond("%s (%s): %s" % (stop['official_name'], stop['area'], stop['routes']))

smsBEST/mumbai/views.py

@@ -0,0 +1 @@
+# Create your views here.

smsBEST/settings.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# vim: ai ts=4 sts=4 et sw=4
+# encoding=utf-8
+
+# -------------------------------------------------------------------- #
+#                          MAIN CONFIGURATION                          #
+# -------------------------------------------------------------------- #
+
+
+# you should configure your database here before doing any real work.
+# see: http://docs.djangoproject.com/en/dev/ref/settings/#databases
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql_psycopg2",
+        "NAME": "smsbest",
+    }
+}
+
+
+# the rapidsms backend configuration is designed to resemble django's
+# database configuration, as a nested dict of (name, configuration).
+#
+# the ENGINE option specifies the module of the backend; the most common
+# backend types (for a GSM modem or an SMPP server) are bundled with
+# rapidsms, but you may choose to write your own.
+#
+# all other options are passed to the Backend when it is instantiated,
+# to configure it. see the documentation in those modules for a list of
+# the valid options for each.
+INSTALLED_BACKENDS = {
+    #"att": {
+    #    "ENGINE": "rapidsms.backends.gsm",
+    #    "PORT": "/dev/ttyUSB0"
+    #},
+    #"verizon": {
+    #    "ENGINE": "rapidsms.backends.gsm,
+    #    "PORT": "/dev/ttyUSB1"
+    #},
+    "atlas": {
+        "ENGINE": "rapidsms.backends.http",
+	"host": "0.0.0.0",
+        "port": 8086,
+        "gateway_url": "http://atlas.gnowledge.org:8001/gateway/send",
+        "params_outgoing": "secret=something+secret&to=%(phone_number)s&txt=%(message)s",
+        "params_incoming": "from=%(phone_number)s&txt=%(message)s"
+
+    },
+    "message_tester": {
+        "ENGINE": "rapidsms.backends.bucket",
+    }
+}
+
+
+# to help you get started quickly, many django/rapidsms apps are enabled
+# by default. you may wish to remove some and/or add your own.
+INSTALLED_APPS = [
+
+    # the essentials.
+    "django_nose",
+    "djtables",
+    "rapidsms",
+
+    # common dependencies (which don't clutter up the ui).
+    "rapidsms.contrib.handlers",
+    "rapidsms.contrib.ajax",
+
+    # enable the django admin using a little shim app (which includes
+    # the required urlpatterns), and a bunch of undocumented apps that
+    # the AdminSite seems to explode without.
+    "django.contrib.sites",
+    "django.contrib.auth",
+    "django.contrib.admin",
+    "django.contrib.sessions",
+    "django.contrib.contenttypes",
+
+    # the rapidsms contrib apps.
+    # "rapidsms.contrib.default",
+    # "rapidsms.contrib.export",
+    "rapidsms.contrib.httptester",
+    # "rapidsms.contrib.locations",
+    "rapidsms.contrib.messagelog",
+    "rapidsms.contrib.messaging",
+    "rapidsms.contrib.registration",
+    # "rapidsms.contrib.scheduler",
+    # "rapidsms.contrib.echo",
+
+    "mumbai"
+]
+
+
+
+# this rapidsms-specific setting defines which views are linked by the
+# tabbed navigation. when adding an app to INSTALLED_APPS, you may wish
+# to add it here, also, to expose it in the rapidsms ui.
+RAPIDSMS_TABS = [
+    ("rapidsms.contrib.messagelog.views.message_log",       "Message Log"),
+    ("rapidsms.contrib.registration.views.registration",    "Registration"),
+    ("rapidsms.contrib.messaging.views.messaging",          "Messaging"),
+    # ("rapidsms.contrib.locations.views.locations",          "Map"),
+    # ("rapidsms.contrib.scheduler.views.index",              "Event Scheduler"),
+    ("rapidsms.contrib.httptester.views.generate_identity", "Message Tester"),
+]
+
+
+# -------------------------------------------------------------------- #
+#                         BORING CONFIGURATION                         #
+# -------------------------------------------------------------------- #
+
+
+# debug mode is turned on as default, since rapidsms is under heavy
+# development at the moment, and full stack traces are very useful
+# when reporting bugs. don't forget to turn this off in production.
+DEBUG = TEMPLATE_DEBUG = True
+
+
+# after login (which is handled by django.contrib.auth), redirect to the
+# dashboard rather than 'accounts/profile' (the default).
+LOGIN_REDIRECT_URL = "/"
+
+
+# use django-nose to run tests. rapidsms contains lots of packages and
+# modules which django does not find automatically, and importing them
+# all manually is tiresome and error-prone.
+TEST_RUNNER = "django_nose.NoseTestSuiteRunner"
+
+
+# for some reason this setting is blank in django's global_settings.py,
+# but it is needed for static assets to be linkable.
+MEDIA_URL = "/static/"
+
+
+# this is required for the django.contrib.sites tests to run, but also
+# not included in global_settings.py, and is almost always ``1``.
+# see: http://docs.djangoproject.com/en/dev/ref/contrib/sites/
+SITE_ID = 1
+
+
+# the default log settings are very noisy.
+LOG_LEVEL = "DEBUG"
+LOG_FILE = "rapidsms.log"
+LOG_FORMAT = "[%(name)s]: %(message)s"
+LOG_SIZE = 8192  # 8192 bits = 8 kb
+LOG_BACKUPS = 256  # number of logs to keep
+
+
+# these weird dependencies should be handled by their respective apps,
+# but they're not, so here they are. most of them are for django admin.
+TEMPLATE_CONTEXT_PROCESSORS = [
+    "django.core.context_processors.auth",
+    "django.core.context_processors.debug",
+    "django.core.context_processors.i18n",
+    "django.core.context_processors.media",
+    "django.core.context_processors.request",
+]
+
+# template loaders load templates from various places.
+# for djtables to work properly, the egg loader needs to be
+# included, the others are fairly standard.
+TEMPLATE_LOADERS = (
+                    'django.template.loaders.filesystem.Loader',
+                    'django.template.loaders.app_directories.Loader',
+                    'django.template.loaders.eggs.Loader'
+)
+
+# -------------------------------------------------------------------- #
+#                           HERE BE DRAGONS!                           #
+#        these settings are pure hackery, and will go away soon        #
+# -------------------------------------------------------------------- #
+
+
+# these apps should not be started by rapidsms in your tests, however,
+# the models and bootstrap will still be available through django.
+TEST_EXCLUDED_APPS = [
+    "django.contrib.sessions",
+    "django.contrib.contenttypes",
+    "django.contrib.auth",
+    "rapidsms",
+    "rapidsms.contrib.ajax",
+    "rapidsms.contrib.httptester",
+]
+
+# the project-level url patterns
+ROOT_URLCONF = "urls"
+
+# import local_settings.py
+try:
+    from local_settings import *
+except:
+    pass
+
+# since we might hit the database from any thread during testing, the
+# in-memory sqlite database isn't sufficient. it spawns a separate
+# virtual database for each thread, and syncdb is only called for the
+# first. this leads to confusing "no such table" errors. We create
+# a named temporary instance instead.
+import os
+import tempfile
+import sys
+
+if 'test' in sys.argv:
+    for db_name in DATABASES:
+        DATABASES[db_name]['TEST_NAME'] = os.path.join(
+            tempfile.gettempdir(),
+            "%s.rapidsms.test.sqlite3" % db_name)
+

smsBEST/urls.py

@@ -0,0 +1,37 @@
+from django.conf.urls.defaults import *
+from django.conf import settings
+from django.contrib import admin
+
+admin.autodiscover()
+
+urlpatterns = patterns('',
+    # Example:
+    # (r'^my-project/', include('my_project.foo.urls')),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    (r'^admin/', include(admin.site.urls)),
+    
+    # RapidSMS core URLs
+    (r'^account/', include('rapidsms.urls.login_logout')),
+    url(r'^$', 'rapidsms.views.dashboard', name='rapidsms-dashboard'),
+
+    # RapidSMS contrib app URLs
+    (r'^ajax/', include('rapidsms.contrib.ajax.urls')),
+    (r'^export/', include('rapidsms.contrib.export.urls')),
+    (r'^httptester/', include('rapidsms.contrib.httptester.urls')),
+    (r'^locations/', include('rapidsms.contrib.locations.urls')),
+    (r'^messagelog/', include('rapidsms.contrib.messagelog.urls')),
+    (r'^messaging/', include('rapidsms.contrib.messaging.urls')),
+    (r'^registration/', include('rapidsms.contrib.registration.urls')),
+    (r'^scheduler/', include('rapidsms.contrib.scheduler.urls')),
+)
+
+if settings.DEBUG:
+    urlpatterns += patterns('',
+        # helper URLs file that automatically serves the 'static' folder in
+        # INSTALLED_APPS via the Django static media server (NOT for use in
+        # production)
+        (r'^', include('rapidsms.urls.static_media')),
+    )