CAMP/manifests
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove json from tree

Browse Source
This commit is contained in:
Sanj 2011-12-08 02:07:30 +05:30
parent a066a625f6
commit aad194dceb
14 changed files with 12417 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
manifests/__init__.py Normal file
View File

11721
manifests/goodErrors.json Normal file
View File

File diff suppressed because it is too large Load Diff

11
manifests/manage.py Executable file
View File

@ -0,0 +1,11 @@
#!/usr/bin/env python
from django.core.management import execute_manager
try:
import settings # Assumed to be in the same directory.
except ImportError:
import sys
sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
sys.exit(1)
if __name__ == "__main__":
execute_manager(settings)

1
manifests/manifestsErrors.json Normal file
View File

@ -0,0 +1 @@
[]

104
manifests/settings.py Normal file
View File

@ -0,0 +1,104 @@
# Django settings for manifests project.
import os
from os.path import join
DEBUG = True
TEMPLATE_DEBUG = DEBUG
JSON_DEBUG = DEBUG
ADMINS = (
# ('Your Name', 'your_email@domain.com'),
)
PROJECT_PATH = os.path.dirname(__file__)
DATA_PATH = join(PROJECT_PATH, "data")
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'manifests', # Or path to database file if using sqlite3.
'USER': 'sanj', # Not used with sqlite3.
'PASSWORD': '', # Not used with sqlite3.
'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
'PORT': '', # Set to empty string for default. Not used with sqlite3.
}
}
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True
# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = join(PROJECT_PATH, "static")
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = '/static/'
# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'
# Make this unique, and don't share it with anybody.
SECRET_KEY = '#7w*kxhnvpvq)v0fj(%1+rqphvzmjp%i9xfi=4&%*3!*&(b#v2'
# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
'django.template.loaders.eggs.Loader',
)
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
)
ROOT_URLCONF = 'manifests.urls'
TEMPLATE_DIRS = (
join(PROJECT_PATH, 'templates')
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
)
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'ships',
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
'django.contrib.admindocs',
)

273
manifests/shipErrors.json Normal file
View File

@ -0,0 +1,273 @@
[
[
"Export",
"1030905",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1006240",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1006474",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1006478",
"",
"",
"",
"",
"",
""
],
[
"Export",
"0",
"",
"",
"",
"",
"",
""
],
[
"Export",
"0",
"",
"",
"",
"",
"",
""
],
[
"Export",
"0",
"",
"",
"",
"",
"",
""
],
[
"Export",
"3",
"BGS",
"600",
"",
"",
"\u0627\u0644\u0646\u0648\u062e\u0630\u0629 \u0645\u0639 \u0627\u0644\u0628\u062d\u0627\u0631\u0629",
""
],
[
"Export",
"10",
"",
"",
"",
"",
"",
""
],
[
"Export",
"12",
"",
"",
"",
"",
"",
""
],
[
"Export",
"7765",
"VEH",
"1",
"1795",
"",
"",
""
],
[
"Export",
"17423",
"PCS",
"195",
"15000",
"",
"\u0639\u0628\u062f\u0627\u0644\u0633\u0644\u0627\u0645 \u0639\u0628\u062f\u0627\u0644\u0639\u0632\u064a\u0632",
""
],
[
"Export",
"13590",
"BGS",
"600",
"24000",
"",
"\u064a\u0639\u0642\u0648\u0628 \u0628\u0627\u0632\u0645\u0627\u0646\u062f\u0643\u0627\u0646",
""
],
[
"Export",
"1008258",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008141",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008138",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008143",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008257",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008147",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008260",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008261",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008262",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008146",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008244",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008395",
"",
"",
"",
"",
"",
""
],
[
"Export",
"1008398",
"",
"",
"",
"",
"",
""
],
[
"Export",
"9",
"5/8 POSTRIEVE PACKER (OIL WELL EQUIPMENT) DATAILS AS PER ATTACHED INV NO . DDAO /2026 /2008",
"PKGS",
"1",
"500",
"",
"",
""
]
]

0
manifests/ships/__init__.py Normal file
View File

20
manifests/ships/admin.py Normal file
View File

@ -0,0 +1,20 @@
from django.contrib import admin
from models import *
class ManifestAdmin(admin.ModelAdmin):
pass
class GoodInline(admin.TabularInline):
model = Good
extra = 10
class ShipAdmin(admin.ModelAdmin):
inlines = [GoodInline]
class GoodAdmin(admin.ModelAdmin):
pass
admin.site.register(Manifest, ManifestAdmin)
admin.site.register(Good, GoodAdmin)
admin.site.register(Ship, ShipAdmin)

200
manifests/ships/convert.py Executable file
View File

@ -0,0 +1,200 @@
import codecs
from glob import glob
import sys
import os
import csv
import json
def toCsv():
manifests = glob('*.txt')
for m in manifests:
f = codecs.open(m, "r", encoding="windows-1256")
txt = unicode(f.read())
f.close()
outFileName = m.replace("txt", "csv")
print outFileName
w = codecs.open(outFileName, "w", "utf-8")
w.write(txt)
w.close()
print "done"
def toRaw():
csvs = glob("*.csv")
for c in csvs:
outFile = c.replace("csv", "html")
print outFile
f = codecs.open(outFile, "w", "utf-8")
r = csv.reader(open(c))
f.write("<table>")
# print "<table>"
for row in r:
f.write("<tr>")
for col in row:
f.write("<td>" + col.decode("utf-8") + "</td>")
f.write("</tr>")
f.write("</table>")
f.close()
print "done"
def toHtml():
htmls = glob("raw/*.html")
for h in htmls:
outFile = h.replace("raw/", "html/")
print outFile
w = codecs.open(outFile, "w", "utf-8")
header = open("header.html").read()
footer = open("footer.html").read()
f = codecs.open(h, "r", encoding="utf-8").read()
w.write(header + f + footer)
w.close()
print "done"
def isShip(row):
try:
# if str(int(row[0])) == row[0]:
# return True
if row[0].strip().lower() == 'import' or row[0].strip().lower() == 'rexport':
return True
else:
return False
except:
return False
def csvToJSON(prefix):
csvs = glob(prefix + "*.csv")
d = []
for c in csvs:
filename = c
date = filename.replace(prefix, "")
data = {
'filename': filename,
'date': date,
'ships': []
}
r = csv.reader(open(c))
rows = []
for row in r:
rows.append(row)
i = 0
ships = []
while i < len(rows):
thisRow = rows[i]
if isShip(thisRow): #is a ship
print thisRow[0]
thisShip = {
'row': thisRow,
'goods': []
}
i += 1
if i == len(rows):
break
currRow = rows[i]
while isShip(currRow) == False:
goodRow = currRow
thisShip['goods'].append(goodRow)
i += 1
if i == len(rows):
break
currRow = rows[i]
ships.append(thisShip)
data['ships'] = ships
d.append(data)
outFile = prefix + "Data.json"
f = codecs.open(outFile, "w", "utf-8")
f.write(json.dumps(d, indent=2))
f.close()
return
def addType(inFile, outFile):
inData = json.loads(open(inFile).read())
outFile = codecs.open(outFile, "w", "utf-8")
for fil in inData:
for ship in fil['ships']:
ship['row'].insert(0, "Export")
print ship['row'][1]
outJSON = json.dumps(inData)
outFile.write(outJSON)
outFile.close()
def importJSON(filename):
from models import Ship, Good, Manifest
data = json.loads(open(filename).read())
errors_ships = []
errors_goods = []
manifests_errors = []
for manifest in data:
m = Manifest()
m.filename = manifest['filename']
month = manifest['date'][0:3]
yr = manifest['date'][3:5]
m.month = month
m.year = int("20" + yr)
# datestring = "20%s-%s" % (yr, month,)
# m.date = datestring
try:
m.save()
except:
manifests_errors.append(manifest['filename'])
for ship in manifest['ships']:
s = Ship()
row = ship['row']
s.manifest_file = m
s.bill_type = row[0]
s.number = row[1]
shipDate = row[2]
s.date = "%s-%s-%s" % (shipDate[6:], shipDate[3:5], shipDate[0:2])
s.ship_name = row[3]
s.captain = row[4]
s.flag = row[5]
s.owner = row[6]
try:
s.port = row[7]
except:
s.port = ''
try:
s.country = row[8]
except:
s.country = ''
try:
s.save()
print "ship success"
except:
print "ship error"
errors_ships.append(row)
if s.id:
for good in ship['goods']:
g = Good()
g.ship = s
g.description = good[0]
g.package_type = good[1]
if good[2].strip() != '':
try:
g.no_of_packages = int(good[2])
except:
errors_goods.append(good)
if good[3].strip() != '':
try:
g.weight = int(good[3])
except:
errors_goods.append(good)
if good[4].strip() != '':
try:
g.value = int(good[4])
except:
errors_goods.append(good)
try:
g.save()
except:
print "goods error"
errors_goods.append(good)
ship_errors_file = open("shipErrors.json", "w")
ship_errors_file.write(json.dumps(errors_ships, indent=2))
ship_errors_file.close()
good_errors_file = open("goodErrors.json", "w")
good_errors_file.write(json.dumps(errors_goods, indent=2))
good_errors_file.close()
manifests_errors_file = open("manifestsErrors.json", "w")
manifests_errors_file.write(json.dumps(manifests_errors, indent=2))
manifests_errors_file.close()

8
manifests/ships/import_data.py Normal file
View File

@ -0,0 +1,8 @@
import json
from settings import DATA_PATH
from os.path import join
json_file_path = join(DATA_PATH, "allData.json")
def importJSON():

39
manifests/ships/models.py Normal file
View File

@ -0,0 +1,39 @@
from django.db import models
class Manifest(models.Model):
month = models.CharField(max_length=3)
year = models.IntegerField(max_length=4)
filename = models.CharField(max_length=256)
def __unicode__(self):
return self.filename
#['type', 'number', 'date', 'ship_name', 'captain', 'flag', 'trader', 'port', 'country']
class Ship(models.Model):
manifest_file = models.ForeignKey(Manifest)
bill_type = models.CharField(max_length=50)
number = models.IntegerField(max_length=20)
date = models.DateField()
ship_name = models.CharField(max_length=255)
captain = models.CharField(max_length=255)
flag = models.CharField(max_length=255)
owner = models.CharField(max_length=255)
port = models.CharField(max_length=255)
country = models.CharField(max_length=255)
def __unicode__(self):
return "%d: %s" % (self.number, self.ship_name,)
#['description', 'package_type', 'no_of_packages', 'weight', 'value']
class Good(models.Model):
ship = models.ForeignKey(Ship)
description = models.TextField()
package_type = models.CharField(max_length=256)
no_of_packages = models.IntegerField(max_length=20, null=True)
weight = models.IntegerField(max_length=20, null=True)
value = models.IntegerField(max_length=20, null=True)
def __unicode__(self):
return self.description
# Create your models here.

23
manifests/ships/tests.py Normal file
View File

@ -0,0 +1,23 @@
"""
This file demonstrates two different styles of tests (one doctest and one
unittest). These will both pass when you run "manage.py test".
Replace these with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.failUnlessEqual(1 + 1, 2)
__test__ = {"doctest": """
Another way to test that 1 + 1 is equal to 2.
>>> 1 + 1 == 2
True
"""}

1
manifests/ships/views.py Normal file
View File

@ -0,0 +1 @@
# Create your views here.

16
manifests/urls.py Normal file
View File

@ -0,0 +1,16 @@
from django.conf.urls.defaults import *
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()
urlpatterns = patterns('',
# Example:
# (r'^manifests/', include('manifests.foo.urls')),
# Uncomment the admin/doc line below to enable admin documentation:
# (r'^admin/doc/', include('django.contrib.admindocs.urls')),
# Uncomment the next line to enable the admin:
(r'^admin/', include(admin.site.urls)),
)