Changes to gtfs, etc

chaloBEST/gtfs/gtfs_export.py

@@ -8,7 +8,7 @@ import datetime
 from itertools import dropwhile
 import copy    
 from fuzzywuzzy import process as fuzzprocess
-
+import pdb
 
 def routeWithLocationData(route):
     '''
@@ -400,7 +400,7 @@ def get_routedetail_subset(unr, direction):
     Algo. 
     1. get routedetail(rd) list
     2. check if rds are reversed
-    3. select stops if they are in UP or DOWN route
+    3. select stops if they are in UP or DOWN route, if a stop is to be removed, transfer the km info to the next or previous stop
     4. get indexes of the stops in the list.
     5. splice the list acc to positions
     6. check if full route, if yes, then ignore calculations, send entire list (only filter up/down stops)
@@ -415,7 +415,7 @@ def get_routedetail_subset(unr, direction):
     route_reversed = False
     # Sometimes to_stop comes before from_stop in RouteDetail.
     # So reverse the list if that happens.. so a from_stop will always come before a to_stop              # This is when the from_stop / to_stop mapping is not proper 
-
+    # reverse list
     for detail in rdlist:
         if detail.stop.id == from_stop.id: break
         if detail.stop.id == to_stop.id:
@@ -428,18 +428,35 @@ def get_routedetail_subset(unr, direction):
     # select stops in route for "UP/DOWN" routes respectively
     if direction in ["UP", "up", "U"]:
         lst = [] 
-        for rd in rdlist:
+        for pos, rd in enumerate(rdlist):
             if rd.stop.dbdirection == '' or rd.stop.dbdirection == 'U' or rd.stop==unr.from_stop or rd.stop==unr.to_stop:
                 lst.append(rd)
+            #else:
+                # add km info of the stop to be removed to next or prev stop, 
+                # if pos != len(rdlist)-1 and rdlist[pos + 1].km:
+                #     rdlist[pos+1].km = rd.km
+                # else:
+                #     if pos != 0 and rdlist[pos - 1].km:
+                #         rdlist[pos+1].km = rd.km
+                    
+                
     else:
         # for "DOWN" routes
         lst = []
-        for rd in rdlist:
+        for pos, rd in enumerate(rdlist):
             if rd.stop.dbdirection == '' or rd.stop.dbdirection ==  'D' or rd.stop==unr.from_stop or rd.stop==unr.to_stop:
                 lst.append(rd)
+            #else:
+                # add km info of the stop to be removed to next or prev stop, 
+                # if pos != len(rdlist)-1 and rdlist[pos + 1].km:
+                #     rdlist[pos+1].km = rd.km
+                # else:
+                #     if pos != 0 and rdlist[pos - 1].km:
+                #         rdlist[pos+1].km = rd.km
+                    
             
     rdlist = lst            
-    
+    pdb.set_trace()
     # 30 lines below only to get the index positions of the from and to stops in the list. 
     from_index = -1
     to_index= -1
@@ -464,9 +481,10 @@ def get_routedetail_subset(unr, direction):
         if(rd.stop==to_stop):
             to_index = rdlist.index(rd)
             to_stop_found+=1            
-            # for ring routes there will be two occurences of the stop, so for 
+            # for ring routes there will be two occurences of the stop, so we want  
-            # remove  break to see multiple occurences in g.multiple_to_stops    
+            # remove  break to see multiple occurences in g.multiple_to_stops            code = unr.route.code[3]
-            break
+            if code == 'R' or code == '4':
+                break
 
     if to_stop_found>1:
         multiple_to_stops.append({"unr":unr,"count":to_stop_found,"to_stop":unr.to_stop})
@@ -476,11 +494,9 @@ def get_routedetail_subset(unr, direction):
         mismatched_unrs['to'].append({"unr":unr,"unr_to_stop_txt":unr.to_stop_txt,"unr_to_stop":unr.to_stop, "route":unr.route})
        
     # indexes found , splice list
-
     rd_subset = rdlist[from_index:to_index+1]
 
     # if ring route
-                    
     if code == 'R' or code == '4':
         # ring specific code here. 
         # converts the given ring route subset to double size.
@@ -522,6 +538,32 @@ def check_route_and_rds():
             lst.add(r)
     return lst
 
+def get_non_ring_routes():
+    """
+    Gets routes which have unique routes which are ring route like but do not have that in route code
+    """
+    unr_lst = set()
+    for unr in UniqueRoute.objects.all():        
+        code = unr.route.code[3]
+        if not (code == 'R' or code == '4'):
+            if unr.from_stop==unr.to_stop:
+                unr_lst.add(unr)
+    return unr_lst
+
+def get_non_ring_routes_via_rds():
+    """
+    Gets routes which have unique routes which are ring route like but do not have that in route code
+    """
+    route_lst = set()
+    for r in Route.objects.all():        
+        code = r.code[3]
+#        if not (code == 'R' or code == '4'):
+#            rds =  
+#            if 
+#                route_lst.add(unr.route)
+    return route_lst
+    
+
 def get_bad_routes():
     """ 
     Gets a list of routes which have less than five routedetails or stops inany of their uniqueroutes.
@@ -1426,19 +1468,28 @@ def export_stops_from_set():
 
 def makeStopList():
     import codecs
-    f =codecs.open(join(PROJECT_ROOT, "gtfs", "Stops.csv"), "w", "utf-8")
+    f =codecs.open(join(PROJECT_ROOT, "../db_csv_files", "MarathiStopsLeft.csv"), "w", "utf-8")
-    f.write("stop_id,stop_code,stop_name,marathi_name")
+    f.write("stop_id\tstop_code\tstop_name\tmarathi_name\n")
 
     for s in Stop.objects.all():
-        if s.name_mr is None:
+        if s.name_mr in ('', None):
             s.name_mr= ''
-        line = str(s.id) +"\t"+ str(s.code)+"\t" + str(s.name) + "\t" + s.name_mr
+            line = str(s.id) +"\t"+ str(s.code)+"\t" + str(s.name) + "\t" + s.name_mr
-        f.write(line+ "\n")
+            f.write(line+ "\n")
 
     f.close()
 
-def get_rd_distance(unr, direction):
+#def readStopList():
-    details= get_routedetail_subset(unr,direction)
+    
+
+def get_rd_distance(unr,addextradist):
+    """
+    Sums up the distances for a given route. 
+    Also adds 300 mts. for each stop for calculating the last stop, 
+    if it doesnt have distance info. (This is for the trip times)
+    def get_rd_distance(UniqueRoute unr, bool addextradist):
+    """
+    details= RouteDetail.objects.filter(route=unr.route).order_by('serial')
     dist =0.0
     for seq, detail in enumerate(details):
         blankstops=0
@@ -1448,21 +1499,30 @@ def get_rd_distance(unr, direction):
         else:
             blankstops+=1
             if seq == len(details) - 1:
-                dist+=float(0.3*blankstops)
+                if addextradist:
+                    dist+=float(0.3*blankstops)
                 
     return dist
 
+def routes_nonstage_last_stop():
+    for r in Route.objects.all():
+        if r.uniqueroute_set.all():
+            unr = r.uniqueroute_set.all()[0]
+            if g.get_rd_distance(unr,0) != g.get_rd_distance(unr,1):
+                cnt.append(r)
+
+    return cnt
+                
 def export_atlas():    
     import codecs
     f =codecs.open(join(PROJECT_ROOT, "gtfs", "recomputed_atlas.csv"), "w", "utf-8")
 
     f = csv.writer(open(join(PROJECT_ROOT, "gtfs", "gtfs_mumbai_bus", "recomputed_atlas.csv"), "w"), delimiter="\t", quotechar='"', quoting=csv.QUOTE_MINIMAL)
-    f.writerow(["RouteCode","RouteAlias","BusesAM","BusesNoon","BusesPM","BusType","Depot","FromStopCode","FromStopName","FromStopOriginal","FirstFrom","LastFrom","ToStopCode","ToStopName","ToStopOriginal","FirstTo","LastTo","rt1","rt2","rt3","rt4","headway1","headway2","headway3","headway4","headway5","ScheduleType","RouteSpan/AtlasDistance", "DistanceMasterRoute", "DistanceRouteDetailUP","DistanceRouteDetailDOWN", "mismatchedfromstop","mismatchedtostop","DaysOfRun" ])
+    f.writerow(["RouteCode","RouteAlias","BusesAM","BusesNoon","BusesPM","BusType","Depot","FromStopCode","FromStopName","FromStopOriginal","FirstFrom","LastFrom","ToStopCode","ToStopName","ToStopOriginal","FirstTo","LastTo","rt1","rt2","rt3","rt4","headway1","headway2","headway3","headway4","headway5","ScheduleType","RouteSpan/AtlasDistance", "DistanceMasterRoute", "DistanceRouteDetail", "mismatchedfromstop","mismatchedtostop","DaysOfRun" ])
     for unr in UniqueRoute.objects.all().order_by("route__code"):
         for rs in unr.routeschedule_set.all().order_by("schedule_type"):
             bus_type= RouteType.objects.get(code=str(unr.route.code)[3]).faretype 
-            dist_up=get_rd_distance(unr,"UP")
+            rd_dist=get_rd_distance(unr,0)
-            dist_down=get_rd_distance(unr,"DOWN")
             
             f.writerow([
                     unr.route.code, 
@@ -1489,8 +1549,7 @@ def export_atlas():
                     rs.schedule_type,
                     unr.distance,
                     unr.route.distance,
-                    dist_up,
+                    rd_dist,
-                    dist_down,
                     1 if 70 > fuzzprocess.ratio(unr.from_stop.name.lower(),unr.from_stop_txt.lower()) else 0,
                     1 if 70 > fuzzprocess.ratio(unr.to_stop.name.lower(),unr.to_stop_txt.lower()) else 0,
                     SCHED[rs.schedule_type].__str__().strip('[]')

chaloBEST/imports/data_mapper.py

@@ -157,9 +157,13 @@ def Stop_save(entry):
 # There is no model as StopMarathi/AreaMarathi, but this is done to separate errors arising from different files, and also that the Marathi names should be done after the Stop and Area entities have been fully loaded cuz thats how we get them from BEST.
 
 def StopMarathi_save(entry):
-    obj = Stop.objects.get(id=int(entry[0])) 
+    obj = Stop.objects.get(code=str(entry[1])) 
-    obj.name_mr = unicode(entry[3], 'utf-8')
+    mrstr =  unicode(entry[3], 'utf-8')
-    obj.save()
+    if mrstr=='':
+        print entry
+    obj.name_mr = mrstr
+    obj.save()    
+    #return obj
     #print obj.__dict__  
 
 def AreaMarathi_save(entry):

chaloBEST/static/js/editstops.js

@@ -39,6 +39,7 @@ var API_BASE = "/1.0/",
         $('.list').click(function(e) {
             var name = $(this).attr("id").replace("sList", ""); //FIXME: stick name in a data attr or so?
             var $target = $(e.target).parent();
+            //console.log($target);
             if (!$target.hasClass('listItem')) {
                 return;
             }
@@ -75,10 +76,12 @@ var API_BASE = "/1.0/",
             $('#stopForm').remove();
             $('#formCol').empty();
             $.getJSON(url, {'srid': 3857}, function(obj) {
+                //console.log("here", $target);
                 $loading.remove();
                 var stopsGeojson = obj.stops;              
                 var stops = stopsGeojson.features;
                 var $stopsList = getStopsList(stops);
+                //console.log($stopsList);
                 var stopsWithGeom = [];
                 $.each(stops, function(i,v) {
                     if (!$.isEmptyObject(v.geometry)) {
@@ -89,11 +92,16 @@ var API_BASE = "/1.0/",
                 stopsGeojson.features = stopsWithGeom;
                 var currFeatures = jsonLayer.features;
                 jsonLayer.removeFeatures(currFeatures);
+                console.log(stopsWithGeom);
                 if (stopsWithGeom.length !== 0) {
                     jsonLayer.addFeatures(geojson_format.read(stopsGeojson));
                     var maxExtent = jsonLayer.getDataExtent();
-                    map.zoomToExtent(maxExtent);                                                                
+                    //What you see here with the setTimeout is a sign of having given up - PLEASE PLEASE make this go away - you should not need to do this - but apparently there is some error being raised by map.zoomToExtent() ALTHOUGH it seems to be doing the right thing and calling it in a setTimeout makes the other code work alright. But it causes me to sleep a little less soundly at night. Please HELP.
-                }                
+                    setTimeout(function() {
+                        map.zoomToExtent(maxExtent);
+                        }, 50);                                                                
+                }
+                //console.log("whoosh", $target);                
                 $target.append($stopsList);
                 // $target.data("hasList", true);
                 $target.data("loading", false);
@@ -315,7 +323,7 @@ var API_BASE = "/1.0/",
         var permalink = new OpenLayers.Control.Permalink({base: "http://www.openstreetmap.org/"});
         map.addControl(permalink);
         $(".olControlPermalink a").attr("target","_blank").html("View in OSM");
-	alert('Not here');
+        //alert('Not here');
     }
 
     function onFeatureSelect(e) {

patches/postgis-adapter-2.patch

@@ -0,0 +1,28 @@
+--- django/contrib/gis/db/backends/postgis/adapter.py.orig	2011-09-09 11:51:27.769648151 +0100
++++ django/contrib/gis/db/backends/postgis/adapter.py	2011-09-12 14:09:51.733962708 +0100
+@@ -12,6 +12,7 @@
+         # the adaptor) and the SRID from the geometry.
+         self.ewkb = str(geom.ewkb)
+         self.srid = geom.srid
++        self._adapter = Binary(self.ewkb)
+ 
+     def __conform__(self, proto):
+         # Does the given protocol conform to what Psycopg2 expects?
+@@ -26,10 +27,15 @@
+     def __str__(self):
+         return self.getquoted()
+ 
++    def prepare(self, conn):
++        # Pass the connection to the adapter: this allows escaping the binary
++        # in the style required by the server's standard_conforming_string setting.
++        self._adapter.prepare(conn)
++
+     def getquoted(self):
+         "Returns a properly quoted string for use in PostgreSQL/PostGIS."
+-        # Want to use WKB, so wrap with psycopg2 Binary() to quote properly.
+-        return 'ST_GeomFromEWKB(E%s)' % Binary(self.ewkb)
++        # psycopg will figure out whether to use E'\\000' or '\000'
++        return 'ST_GeomFromEWKB(%s)' % self._adapter.getquoted()
+ 
+     def prepare_database_save(self, unused):
+         return self