fix_missing changed - algo tweak , data_mapper restored

This commit is contained in:
Johnson Chetty 2012-03-28 13:40:42 +02:00
parent 9a81c6e254
commit dfb01db6e3
3 changed files with 254 additions and 28 deletions

View File

@ -1,6 +1,212 @@
from mumbai.models import *
import csv
from settings import PROJECT_ROOT
from os.path import join
import json
import datetime
import sys
from django.contrib.gis.geos import Point
from imports.import_atlas import getFromToStopsForRoute, importUniqueRoutes
from imports import postload_cleanup as postclean
from decimal import Decimal
globalerr = []
def RouteType_save(entry):
obj = RouteType(code=entry[0], rtype=entry[1], faretype=entry[2])
obj.save()
#print obj.__dict__
def getFromToStopsFromRouteDetails(code):
routeDetails = RouteDetail.objects.filter(route_code=code).order_by('serial')
if routeDetails.count() == 0:
return None
fromStop = routeDetails[0].stop
toStop = routeDetails[routeDetails.count() -1].stop
return (fromStop, toStop,)
def Route_save(entry):
"""
try:
f_stop = Stop.objects.filter(name=str(entry[2]))[0]
except IndexError:
f_stop = None
try:
t_stop = Stop.objects.filter(name=str(entry[3]))[0]
except IndexError:
t_stop = None
"""
from_to = getFromToStopsFromRouteDetails(entry[0])
if from_to is None:
globalerr.append({"data" :entry[0], error:["Route not found"]})
#obj = Route(code=entry[0], alias=entry[1], from_stop_txt=entry[2], to_stop_txt=entry[3], from_stop=from_to[0], to_stop=from_to[1], distance=Decimal(entry[4]), stages=int(entry[5]))
obj = Route(
code=str(entry[0]),
alias=str(entry[1]),
from_stop=from_to[0],
from_stop_txt=str(entry[2]),
to_stop_txt=str(entry[3]),
to_stop=from_to[1],
distance=Decimal(entry[4]),
stages=int(entry[5]))
obj.save()
# throw an error if the stops mapped do not exist.
# but load nulls into db anyway
# IndexError implies that nothing was mapped.
# MultipleRows found indicates ambiguity when there should'nt be.
#f_stop = Stop.objects.get(name=str(entry[2]))[0]
#t_stop = Stop.objects.get(name=str(entry[3]))[0]
#print obj.__dict__
def HardCodedRoute_save(entry):
obj = HardCodedRoute(code=str(entry[0]), alias=entry[1], faretype=entry[2])
obj.save()
#print obj.__dict__
def Depot_save(entry):
obj = Depot(
code=str(entry[0]),
name=str(entry[1]),
stop=int(entry[2])
)
obj.save()
#print obj.__dict__
def Holiday_save(entry):
date_format = entry[0].rsplit('.')
theday = int(date_format[0])
themonth = int(date_format[1])
theyear = int('20'+ date_format[2])
obj = Holiday(date=datetime.date(day=theday, month=themonth, year=theyear), name=str(entry[1]))
obj.save()
#print obj.__dict__
def RouteDetail_save(entry):
temp_stop=Stop.objects.get(code=int(entry[2]))
"""try:
temp_route=Route.objects.get(code=str(entry[0]))
except:
temp_route=None
try:
temp_stop=Stop.objects.get(code=int(entry[2]))
except:
temp_stop=None
"""
obj = RouteDetail(
route_code = entry[0],
route = None,
serial=int(entry[1]),
stop= temp_stop,
stage=(lambda:entry[3].startswith('1'), lambda:None)[ entry[3] == '' ](),
km=(lambda:None,lambda:Decimal(entry[4]))[ entry[4] != '' ]())
obj.save()
#print obj.__dict__
def Road_save(entry):
obj = Road(code=int(entry[0]), name=str(entry[1]))
obj.save()
#print obj.__dict__
def Fare_save(entry):
obj = Fare(
slab=Decimal(entry[0]),
ordinary=int(entry[1]),
limited=int(entry[2]),
express=int(entry[3]),
ac=int(entry[4]),
ac_express=int(entry[5])
)
obj.save()
#print obj.__dict__
def Area_save(entry):
obj = Area(code=int(entry[0]), name= str(entry[1]))
obj.save()
#print obj.__dict__
def Stop_save(entry):
_road = Road.objects.get(code=int(entry[4]))
_area = Area.objects.get(code=int(entry[5]))
try:
_depot = Depot.objects.filter(code=str(entry[6]))[0]
except IndexError:
_depot = None
obj = Stop(
code=int(entry[0]),
name=str(entry[1]),
dbdirection=str(entry[2]),
chowki=(entry[3]).startswith('TRUE'),
road=_road,
area=_area,
depot=_depot
)
obj.save()
#print obj.__dict__
# There is no model as StopMarathi/AreaMarathi, but this is done to separate errors arising from different files, and also that the Marathi names should be done after the Stop and Area entities have been fully loaded cuz thats how we get them from BEST.
def StopMarathi_save(entry):
obj = Stop.objects.get(code=int(entry[0]))
obj.name_mr = unicode(entry[1], 'utf-8')
obj.save()
#print obj.__dict__
def AreaMarathi_save(entry):
obj = Area.objects.get(code=int(entry[0]))
obj.name_mr = unicode(entry[1], 'utf-8')
obj.save()
#print obj.__dict__
loc1s = 0
loc2s = 0
class NoPointsFoundError(Exception):
pass
def StopLocation_save(entry):
this_stop = Stop.objects.get(code=int(entry[4]))
#hits = {'one':[],'two':[],'three':[],'four':[]}
flagerr = 0
if entry[0] and entry[1]:
loc1 = StopLocation(stop=this_stop, point=Point(float(entry[1]), float(entry[0])),direction='U' )
loc1.save()
#loc1s+=1
else:
flagerr=1
if entry[2] and entry[3]:
loc2 = StopLocation(stop=this_stop, point=Point(float(entry[3]), float(entry[2])),direction='D' )
loc2.save()
#loc2s+=1
else:
flagerr+=1
if flagerr == 2:
flagerr = 0
raise NoPointsFoundError
#print "Loc1s found : ", loc1s
#print "Loc2s found : ", loc2s
saveorder = ["Fare","Holiday","Area","Road","Depot","Stop", "StopMarathi","AreaMarathi","RouteDetail", "Route","RouteType","HardCodedRoute"]
saveorder = ["Fare","Holiday","Area","Road","Depot","Stop", "StopMarathi","AreaMarathi","RouteDetail", "Route","RouteType","HardCodedRoute","StopLocation" ]
mappingtosave = { mappingtosave = {
"Fare":Fare_save, "Fare":Fare_save,
@ -14,10 +220,11 @@ mappingtosave = {
"RouteType":RouteType_save, "RouteType":RouteType_save,
"HardCodedRoute":HardCodedRoute_save, "HardCodedRoute":HardCodedRoute_save,
"StopMarathi":StopMarathi_save, "StopMarathi":StopMarathi_save,
"AreaMarathi":AreaMarathi_save "AreaMarathi":AreaMarathi_save,
} "StopLocation":StopLocation_save
# There is no model as StopMarathi/AreaMarathi, but this is done to separate errors arising from different input files. # There is no model as StopMarathi/AreaMarathi, but this is done to separate errors arising from different input files.
}
def loadFKinRouteDetail(): def loadFKinRouteDetail():
err=[] err=[]
@ -94,7 +301,7 @@ def fire_up():
# also # also
importUniqueRoutes() importUniqueRoutes()
print "loading UniqueRoute..." print "loading UniqueRoute..."
postclean.copydefaultStopLocations() #postclean.copydefaultStopLocations()
postclean.copynames2display_name() postclean.copynames2display_name()
#---------------------------------------------------------- #----------------------------------------------------------

View File

@ -4,27 +4,45 @@ from mumbai.models import *
def fix_distances(): def fix_distances():
for unique_route in UniqueRoute.objects.all(): for unique_route in UniqueRoute.objects.all():
# RouteDetail sometimes isn't order from from_stop to to_stop # RouteDetail sometimes isn't in order from from_stop to to_stop
from_stop, to_stop = unique_route.from_stop.id, unique_route.to_stop.id from_stop, to_stop = unique_route.from_stop.id, unique_route.to_stop.id
details = list(unique_route.route.routedetail_set.all()) details = list(unique_route.route.routedetail_set.all())
# Sometimes to_stop comes before from_stop in RouteDetail. What is there to say. # Sometimes to_stop comes before from_stop in RouteDetail. What is there to say.
for detail in details: for detail in details:
if detail.stop.id == from_stop: break if detail.stop.id == from_stop: break
if detail.stop.id == to_stop: if detail.stop.id == to_stop:
details.reverse() details.reverse()
break break
distance = 0.0 distance = 0.0
record = False record = False
for detail in details: last_stop_passed = False
for detail in details:
# basic idea, run thru each detail, if it has km info, then add it, if to_stop reached, and if it did not have km info, then go to the next detail having km info add it and done.
# For route 240RING, some detail.km is null??? # For route 240RING, some detail.km is null???
if record and detail.km: distance += float(detail.km)
# distance > 0 because of 100RING returning 1 stop shy of its start # is a stage
if record and distance > 0 and detail.stop.id == to_stop: if record and detail.km:
record = False if not last_stop_passed:
break distance += float(detail.km)
else:
# if stage having km info reached after last stop, then add and exit loop
distance += float(detail.km)
record=False
last_stop_passed = False
break
# distance > 0 because of 100RING returning 1 stop shy of its start
if distance > 0 and detail.stop.id == to_stop:
last_stop_passed = True
#if record and distance > 0 and detail.stop.id == to_stop and last_stop_reached:
# record = False
# break
# Start recording *after* we check for the break, because, # Start recording *after* we check for the break, because,
# if from_stop == to_stop, we don't want to break on the first stop # if from_stop == to_stop, we don't want to break on the first stop
if detail.stop.id == from_stop: record = True if detail.stop.id == from_stop: record = True
if record: if record:
print Exception("UniqueRoute %s from %s to %s ran off the end while measuring distance!" %(unique_route, unique_route.from_stop.code, unique_route.to_stop.code)) print Exception("UniqueRoute %s from %s to %s ran off the end while measuring distance!" %(unique_route, unique_route.from_stop.code, unique_route.to_stop.code))
if not distance: if not distance:
@ -47,6 +65,22 @@ def fix_missing_runtimes():
for col_idx, column in enumerate(columns): for col_idx, column in enumerate(columns):
# if the runtime is set, AWESOME, bail # if the runtime is set, AWESOME, bail
if getattr(schedule, column): continue if getattr(schedule, column): continue
# try to use the previous column....if available
if getattr(schedule, column): continue
if col_idx > 0:
prev_runtime = getattr(schedule, columns[col_idx-1])
if prev_runtime:
setattr(schedule, column, prev_runtime)
continue
# ... or the next column, if it comes to that.
if col_idx < len(columns)-1:
next_runtime = getattr(schedule, columns[col_idx+1])
if next_runtime:
setattr(schedule, column, next_runtime)
continue
# otherwise, go through the other schedules for this subroute and # otherwise, go through the other schedules for this subroute and
# see if we get a matching runtime -- if so, use it # see if we get a matching runtime -- if so, use it
for sibling in sibling_schedules: for sibling in sibling_schedules:
@ -92,20 +126,5 @@ def fix_missing_runtimes():
setattr(schedule, column, partial_runtime) setattr(schedule, column, partial_runtime)
break break
# OTHER-otherwise, use the previous column....
if getattr(schedule, column): continue
if col_idx > 0:
prev_runtime = getattr(schedule, columns[col_idx-1])
if prev_runtime:
setattr(schedule, column, prev_runtime)
continue
# ... or the next column, if it comes to that.
if col_idx < len(columns)-1:
next_runtime = getattr(schedule, columns[col_idx+1])
if next_runtime:
setattr(schedule, column, next_runtime)
continue
if column != "runtime4": if column != "runtime4":
print Exception("ERR fix_missing_runtimes: %s STILL missing %s!" % (schedule, column)) print Exception("ERR fix_missing_runtimes: %s STILL missing %s!" % (schedule, column))

View File

@ -353,7 +353,7 @@ class StopLocation(models.Model):
class Depot(models.Model): class Depot(models.Model):
code = models.CharField(max_length=5, unique=True) code = models.CharField(max_length=5) # should have unique=True ?
name = models.TextField(max_length=50) name = models.TextField(max_length=50)
stop = models.IntegerField() stop = models.IntegerField()