atlas import, almost there

This commit is contained in:
Sanj 2012-01-11 02:30:55 +05:30
parent 032f350ac2
commit eae4d52bec
4 changed files with 264 additions and 12 deletions

Binary file not shown.

View File

@ -2,6 +2,10 @@ from settings import PROJECT_ROOT
from os.path import join from os.path import join
import json import json
import csv import csv
import pdb
from mumbai.models import *
from fuzzywuzzy import process as fuzzprocess
import datetime
#Get levenshtein distance between two strings, from http://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance #Get levenshtein distance between two strings, from http://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance
def levenshtein(s1, s2): def levenshtein(s1, s2):
@ -38,6 +42,252 @@ def csvToJSON():
jsonFile.write(json.dumps(atlasDict, indent=2)) jsonFile.write(json.dumps(atlasDict, indent=2))
jsonFile.close() jsonFile.close()
'''
function to copy over values of AM N PM + Schedule from previous row, reading from Atlas.json, writing to atlasCopied.json
'''
def processJSON():
routeErrors = {'routes': [], 'others': []}
routeMapping = json.loads(open(join(PROJECT_ROOT, "../db_csv_files/routeMapping.json")).read())
routes = json.loads(open(join(PROJECT_ROOT, "../db_csv_files/Atlas.json")).read())
previousRow = []
outDict = {}
for key in routes.keys():
print key
if key not in routeMapping: #make note of routeNames we dont have routeAlias for yet.
routeErrors['routes'].append(key)
else: #else, go ahead ..
routeAlias = routeMapping[key]
thisRoute = routes[key]
#handle copying over empty values from previous rows
outDict[key] = []
for row in thisRoute:
# pdb.set_trace()
if len(row) < 7:
routeErrors['others'].append({key: row})
break
for i in range(2,4):
if row[i].strip() == '':
row[i] = previousRow[i]
try:
if row[-5].strip() == '':
row[-5] = previousRow[-5]
except:
pdb.set_trace()
previousRow = row
outDict[key].append(row)
atlasRouteErrors = open("atlasRouteErrors.json", "w")
atlasRouteErrors.write(json.dumps(routeErrors, indent=2))
atlasRouteErrors.close()
atlasCopied = open(join(PROJECT_ROOT, "../db_csv_files/atlasCopied.json"), "w")
atlasCopied.write(json.dumps(outDict, indent=2))
atlasCopied.close()
'''
function to group atlasCopied.json to uniqueRoutes (uniqueRoutes.json)
'''
def groupUnique():
routes = json.loads(open(join(PROJECT_ROOT, "../db_csv_files/atlasCopied.json")).read())
errors = {}
outDict = {}
for key in routes.keys():
outDict[key] = []
for row in routes[key]:
print key
d = {
'from': row[7],
'to': row[10],
'span': row[13],
'is_full': False,
# 'schedule': row[28],
'rows': {
row[-5]: row
}
}
matchedRow = isNotUnique(d, outDict[key])
if matchedRow:
schedule = row[-5]
outDict[key][matchedRow]['rows'][schedule] = row
else:
if isLargestSpan(d, routes[key]):
d['is_full'] = True
outDict[key].append(d)
outFile = open(join(PROJECT_ROOT, "../db_csv_files/uniqueRoutes.json"), "w")
outFile.write(json.dumps(outDict, indent=2))
outFile.close()
'''
Import RouteMaster
'''
def importRouteMaster():
CsvFile = csv.reader(open(join(PROJECT_ROOT, "../db_csv_files/RouteMaster.csv"), "r"), delimiter=',')
test = CsvFile.next()
stop_errors = []
print test
for row in CsvFile:
if len(row) < 1:
continue
from_to = getFromToStopsForRoute(row[0])
if from_to is None:
stop_errors.append(row[0])
continue
print row[0]
obj = Route(code=row[0], alias=row[1], from_stop_txt=row[2], to_stop_txt=row[3], from_stop=from_to[0], to_stop=from_to[1], distance=row[4], stages=int(row[5]))
obj.save()
errors = open(join(PROJECT_ROOT, "../errors/routeStopErrors.json"), "w")
errors.write(json.dumps(stop_errors, indent=2))
errors.close()
def importUniqueRoutes():
data = json.load(open(join(PROJECT_ROOT, "../db_csv_files/uniqueRoutes.json")))
routeMapping = json.load(open(join(PROJECT_ROOT, "../db_csv_files/routeMapping.json")))
routeDoesNotExistErrors = []
stopMapping = {}
stopErrors = []
for route in data.keys():
routeCode = routeMapping[route]
try:
routeObj = Route.objects.get(code=routeCode)
except:
routeDoesNotExistErrors.append({'routeCode': routeCode, 'routeAlias': route})
continue
for unique_route in data[route]:
thisRoute = unique_route #FIXME
try:
distance = float(thisRoute['span'])
except:
distance = 0
obj = UniqueRoute(route=routeObj, is_full=thisRoute['is_full'], distance=distance, from_stop_txt=thisRoute['from'], to_stop_txt=thisRoute['to'])
if obj.is_full:
from_to = getFromToStopsForRoute(routeObj.code)
obj.from_stop = from_to[0]
if not stopMapping.has_key(obj.from_stop_txt):
stopMapping[obj.from_stop_txt] = from_to[0].stopcd
obj.to_stop = from_to[1]
if not stopMapping.has_key(obj.to_stop_txt):
stopMapping[obj.to_stop_txt] = from_to[1].stopcd
else:
stopnames = []
stopcodes = []
for r in RouteDetails.objects.filter(rno=routeObj.code):
stopnames.append(r.stopcd.stopnm)
stopcodes.append(r.stopcd.stopcd)
from_fuzz = fuzzprocess.extractOne(thisRoute['from'], stopnames)
to_fuzz = fuzzprocess.extractOne(thisRoute['to'], stopnames)
#pdb.set_trace()
try:
obj.from_stop = Stop.objects.filter(stopnm=from_fuzz[0]).filter(stopcd__in=stopcodes)[0]
obj.to_stop = Stop.objects.filter(stopnm=to_fuzz[0]).filter(stopcd__in=stopcodes)[0]
except:
stopErrors.append([thisRoute['from'], thisRoute['to']])
continue
obj.save()
#pdb.set_trace()
# print thisRoute['rows'].keys()
for schedule in thisRoute['rows'].keys():
row = thisRoute['rows'][schedule]
try:
depot = Depot.objects.get(depot_code=row[6])
except:
depot = None
#pdb.set_trace()
routeScheduleObj = RouteSchedule(unique_route=obj, schedule_type=schedule, busesAM=noneInt(row[2]), busesN=noneInt(row[3]), busesPM=noneInt(row[4]), bus_type=row[5], depot_txt=row[6], depot=depot, first_from=formatTime(row[8]), last_from=formatTime(row[9]), first_to=formatTime(row[11]), last_to=formatTime(row[12]), runtime1=noneInt(row[14]), runtime2=noneInt(row[15]), runtime3=noneInt(row[16]), runtime4=noneInt(row[17]), headway1=noneInt(row[18]), headway2=noneInt(row[19]), headway3=noneInt(row[20]), headway4=noneInt(row[21]), headway5=noneInt(row[22]))
routeScheduleObj.save()
errors = open(join(PROJECT_ROOT, "../errors/routeMasterMissingRoutes.json"), "w")
errors.write(json.dumps(routeDoesNotExistErrors, indent=2))
errors.close()
stopMappingFile = open(join(PROJECT_ROOT, "../db_csv_files/stopMapping.json"), "w")
stopMappingFile.write(json.dumps(stopMapping, indent=2))
stopMappingFile.close()
stopErrorsFile = open(join(PROJECT_ROOT, "../errors/atlasStopErrors.json"), "w")
stopErrorsFile.write(json.dumps(stopErrors, indent=2))
stopErrorsFile.close()
'''
>>>formatTime("06.40")
>>>time(6,40)
'''
def formatTime(s):
if s.strip() == '':
return datetime.time(0,0)
split = s.split(".")
try:
hour = int(split[0])
except:
hour = 0
try:
minutes = int(split[1])
except:
minutes = 0 #FIXME: ugly!!
#pdb.set_trace()
try:
return datetime.time(hour, minutes)
except:
return datetime.time(0,0)
def noneInt(val):
try:
return int(val)
except:
return None
def getFromToStopsForRoute(routeCode):
# fromStr = row[2]
routeDetails = RouteDetails.objects.filter(rno=routeCode).order_by('stopsr')
if routeDetails.count() == 0:
return None
fromStop = routeDetails[0].stopcd
toStop = routeDetails[routeDetails.count() -1].stopcd
return (fromStop, toStop,)
def isLargestSpan(data, arr):
span = data['span']
for a in arr:
try:
arrSpan = float(a[13])
except:
arrSpan = 0
try:
dataSpan = float(data['span'])
except:
dataSpan = 0
if arrSpan > dataSpan:
return False
return True
'''
returns index of row if not unique, else False
'''
def isNotUnique(data, arr):
i = 0
for a in arr:
if a['from'] == data['from'] and a['to'] == data['to'] and a['span'] == data['span']:
return i
i += 1
return False
def getRouteCodes():
atlasRawCSV = csv.reader(open(join(PROJECT_ROOT, "../db_csv_files/AtlasRaw.csv"), "r"), delimiter="\t")
atlasDict = json.loads(open(join(PROJECT_ROOT, "../db_csv_files/Atlas.json")).read())
mapping = {}
missingFile = open("/tmp/routesMissing.txt", "w")
for row in atlasRawCSV:
routeAlias = row[0]
routeCode = row[1]
print routeCode
if routeCode in atlasDict:
mapping[routeCode] = routeAlias
else:
missingFile.write(routeCode + "\n")
missingFile.close()
mappingFile = open(join(PROJECT_ROOT, "../db_csv_files/routeMapping.json"), "w")
mappingFile.write(json.dumps(mapping, indent=2))
mappingFile.close()

View File

@ -64,7 +64,7 @@ class RouteDetailsAdmin(admin.ModelAdmin):
class RouteAdmin(admin.ModelAdmin): class RouteAdmin(admin.ModelAdmin):
list_display = ("routealias","route","from_stop","to_stop","distance","stages") list_display = ("alias","code","from_stop","to_stop","distance","stages")
formfield_overrides = { formfield_overrides = {
models.TextField: {'widget': forms.TextInput}, models.TextField: {'widget': forms.TextInput},
} }

View File

@ -95,28 +95,29 @@ class RouteDetails(models.Model):
class Route(models.Model): class Route(models.Model):
route = models.TextField(max_length=255) code = models.TextField(max_length=255)
routealias = models.TextField(max_length=255) alias = models.TextField(max_length=255)
from_stop = models.TextField(max_length=500,db_column='from') from_stop_txt = models.TextField(max_length=500,db_column='from')
to_stop = models.TextField(max_length=500,db_column='to') to_stop_txt = models.TextField(max_length=500,db_column='to')
# Ideally they should be ... from_stop = models.ForeignKey(Stop, related_name='from_stop')
#from_stop = models.ForeignKey(Stop, related_name='from_stop') to_stop = models.ForeignKey(Stop, related_name='to_stop')
#to_stop = models.ForeignKey(Stop, related_name='to_stop')
distance = models.DecimalField(max_digits=3, decimal_places=1) distance = models.DecimalField(max_digits=3, decimal_places=1)
stages = models.IntegerField() stages = models.IntegerField()
def __unicode__(self): def __unicode__(self):
return self.route return self.alias
class UniqueRoute(models.Model): class UniqueRoute(models.Model):
route = models.ForeignKey(Route) route = models.ForeignKey(Route)
from_stop_txt = models.CharField(max_length=255)
to_stop_txt = models.CharField(max_length=255)
from_stop = models.ForeignKey(Stop, related_name="routes_from") from_stop = models.ForeignKey(Stop, related_name="routes_from")
to_stop = models.ForeignKey(Stop, related_name="routes_to") to_stop = models.ForeignKey(Stop, related_name="routes_to")
distance = models.DecimalField(max_digits=3, decimal_places=2) distance = models.FloatField(blank=True, null=True)
is_full = models.BooleanField() is_full = models.BooleanField()
def __unicode__(self): def __unicode__(self):
return "%s: %s to %s" % (self.route.routealias, self.from_stop, self.to_stop,) return "%s: %s to %s" % (self.route.alias, self.from_stop, self.to_stop,)
class RouteSchedule(models.Model): class RouteSchedule(models.Model):
unique_route = models.ForeignKey(UniqueRoute) unique_route = models.ForeignKey(UniqueRoute)
@ -125,7 +126,8 @@ class RouteSchedule(models.Model):
busesN = models.IntegerField(blank=True, null=True) busesN = models.IntegerField(blank=True, null=True)
busesPM = models.IntegerField(blank=True, null=True) busesPM = models.IntegerField(blank=True, null=True)
bus_type = models.CharField(max_length=3, default="SD") bus_type = models.CharField(max_length=3, default="SD")
depot = models.ForeignKey("Depot") depot_txt = models.CharField(max_length=16, blank=True)
depot = models.ForeignKey("Depot", null=True)
first_from = models.TimeField(blank=True, null=True) first_from = models.TimeField(blank=True, null=True)
last_from = models.TimeField(blank=True, null=True) last_from = models.TimeField(blank=True, null=True)
first_to = models.TimeField(blank=True, null=True) first_to = models.TimeField(blank=True, null=True)