remove json from tree

manifests/manage.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+from django.core.management import execute_manager
+try:
+    import settings # Assumed to be in the same directory.
+except ImportError:
+    import sys
+    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
+    sys.exit(1)
+
+if __name__ == "__main__":
+    execute_manager(settings)

manifests/manifestsErrors.json

@@ -0,0 +1 @@
+[]

manifests/settings.py

@@ -0,0 +1,104 @@
+# Django settings for manifests project.
+import os
+from os.path import join
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+JSON_DEBUG = DEBUG
+
+ADMINS = (
+    # ('Your Name', 'your_email@domain.com'),
+)
+
+PROJECT_PATH = os.path.dirname(__file__)
+DATA_PATH = join(PROJECT_PATH, "data")
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME': 'manifests',                      # Or path to database file if using sqlite3.
+        'USER': 'sanj',                      # Not used with sqlite3.
+        'PASSWORD': '',                  # Not used with sqlite3.
+        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
+        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+    }
+}
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# On Unix systems, a value of None will cause Django to use the same
+# timezone as the operating system.
+# If running in a Windows environment this must be set to the same as your
+# system time zone.
+TIME_ZONE = 'America/Chicago'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale
+USE_L10N = True
+
+# Absolute path to the directory that holds media.
+# Example: "/home/media/media.lawrence.com/"
+MEDIA_ROOT = join(PROJECT_PATH, "static") 
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash if there is a path component (optional in other cases).
+# Examples: "http://media.lawrence.com", "http://example.com/media/"
+MEDIA_URL = '/static/'
+
+# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
+# trailing slash.
+# Examples: "http://foo.com/media/", "/media/".
+ADMIN_MEDIA_PREFIX = '/media/'
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '#7w*kxhnvpvq)v0fj(%1+rqphvzmjp%i9xfi=4&%*3!*&(b#v2'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+     'django.template.loaders.eggs.Loader',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+)
+
+ROOT_URLCONF = 'manifests.urls'
+
+TEMPLATE_DIRS = (
+    join(PROJECT_PATH, 'templates')
+		# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'ships',
+    # Uncomment the next line to enable the admin:
+    'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    'django.contrib.admindocs',
+)

manifests/shipErrors.json

@@ -0,0 +1,273 @@
+[
+  [
+    "Export", 
+    "1030905", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1006240", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1006474", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1006478", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "0", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "0", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "0", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "3", 
+    "BGS", 
+    "600", 
+    "", 
+    "", 
+    "\u0627\u0644\u0646\u0648\u062e\u0630\u0629 \u0645\u0639 \u0627\u0644\u0628\u062d\u0627\u0631\u0629", 
+    ""
+  ], 
+  [
+    "Export", 
+    "10", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "12", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "7765", 
+    "VEH", 
+    "1", 
+    "1795", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "17423", 
+    "PCS", 
+    "195", 
+    "15000", 
+    "", 
+    "\u0639\u0628\u062f\u0627\u0644\u0633\u0644\u0627\u0645 \u0639\u0628\u062f\u0627\u0644\u0639\u0632\u064a\u0632", 
+    ""
+  ], 
+  [
+    "Export", 
+    "13590", 
+    "BGS", 
+    "600", 
+    "24000", 
+    "", 
+    "\u064a\u0639\u0642\u0648\u0628 \u0628\u0627\u0632\u0645\u0627\u0646\u062f\u0643\u0627\u0646", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008258", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008141", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008138", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008143", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008257", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008147", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008260", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008261", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008262", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008146", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008244", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008395", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "1008398", 
+    "", 
+    "", 
+    "", 
+    "", 
+    "", 
+    ""
+  ], 
+  [
+    "Export", 
+    "9", 
+    "5/8 POSTRIEVE PACKER (OIL WELL EQUIPMENT) DATAILS AS PER ATTACHED INV NO . DDAO /2026 /2008", 
+    "PKGS", 
+    "1", 
+    "500", 
+    "", 
+    "", 
+    ""
+  ]
+]

manifests/ships/admin.py

@@ -0,0 +1,20 @@
+from django.contrib import admin
+from models import *
+
+class ManifestAdmin(admin.ModelAdmin):
+    pass
+
+class GoodInline(admin.TabularInline):
+    model = Good
+    extra = 10
+
+class ShipAdmin(admin.ModelAdmin):
+    inlines = [GoodInline]
+
+class GoodAdmin(admin.ModelAdmin):
+    pass
+
+
+admin.site.register(Manifest, ManifestAdmin)
+admin.site.register(Good, GoodAdmin)
+admin.site.register(Ship, ShipAdmin)

manifests/ships/convert.py

@@ -0,0 +1,200 @@
+import codecs
+from glob import glob
+import sys
+import os
+import csv
+import json
+
+def toCsv():
+  manifests = glob('*.txt')
+  for m in manifests:
+    f = codecs.open(m, "r", encoding="windows-1256")
+    txt = unicode(f.read())
+    f.close()    
+    outFileName = m.replace("txt", "csv")
+    print outFileName
+    w = codecs.open(outFileName, "w", "utf-8")
+    w.write(txt)
+    w.close()
+  print "done"
+  
+def toRaw():
+  csvs = glob("*.csv")
+  for c in csvs:
+    outFile = c.replace("csv", "html")
+    print outFile
+    f = codecs.open(outFile, "w", "utf-8")
+    r = csv.reader(open(c))
+    f.write("<table>")
+#    print "<table>"
+    for row in r:
+      f.write("<tr>")
+      for col in row:
+        f.write("<td>" + col.decode("utf-8") + "</td>")
+      f.write("</tr>")
+    f.write("</table>")
+    f.close()
+  print "done"
+
+def toHtml():
+  htmls = glob("raw/*.html")
+  for h in htmls:
+    outFile = h.replace("raw/", "html/")
+    print outFile
+    w = codecs.open(outFile, "w", "utf-8")
+    header = open("header.html").read()
+    footer = open("footer.html").read()
+    f = codecs.open(h, "r", encoding="utf-8").read()
+    w.write(header + f + footer)
+    w.close()
+  print "done"
+    
+
+def isShip(row):
+  try: 
+#    if str(int(row[0])) == row[0]:
+#      return True
+    if row[0].strip().lower() == 'import' or row[0].strip().lower() == 'rexport':
+      return True
+    else:
+      return False
+  except:
+    return False
+
+
+def csvToJSON(prefix):
+  csvs = glob(prefix + "*.csv")
+  d = []
+  for c in csvs:
+    filename = c
+    date = filename.replace(prefix, "")
+    data = {
+      'filename': filename,
+      'date': date,
+      'ships': []
+    }
+    r = csv.reader(open(c))
+    rows = []
+    for row in r:
+      rows.append(row)
+    i = 0
+    ships = []
+    while i < len(rows):
+      thisRow = rows[i]
+      if isShip(thisRow): #is a ship
+        print thisRow[0]
+        thisShip = {
+          'row': thisRow,
+          'goods': []
+        }
+        i += 1
+        if i == len(rows):
+          break
+        currRow = rows[i]       
+        while isShip(currRow) == False:
+          goodRow = currRow
+          thisShip['goods'].append(goodRow)
+          i += 1
+          if i == len(rows):
+            break
+          currRow = rows[i]
+        ships.append(thisShip)
+    data['ships'] = ships
+    d.append(data)
+  outFile = prefix + "Data.json"
+  f = codecs.open(outFile, "w", "utf-8")
+  f.write(json.dumps(d, indent=2))     
+  f.close()
+  return
+
+def addType(inFile, outFile):
+  inData = json.loads(open(inFile).read())
+  outFile = codecs.open(outFile, "w", "utf-8")
+  for fil in inData:
+    for ship in fil['ships']:
+      ship['row'].insert(0, "Export")
+      print ship['row'][1]
+  outJSON = json.dumps(inData)
+  outFile.write(outJSON)
+  outFile.close()
+
+def importJSON(filename):
+  from models import Ship, Good, Manifest
+  data = json.loads(open(filename).read())
+  errors_ships = []
+  errors_goods = []
+  manifests_errors = []
+  for manifest in data:
+    m = Manifest()
+    m.filename = manifest['filename']
+    month = manifest['date'][0:3]
+    yr = manifest['date'][3:5]
+    m.month = month
+    m.year = int("20" + yr)
+#    datestring = "20%s-%s" % (yr, month,)
+#    m.date = datestring
+    try:
+      m.save()
+    except:
+      manifests_errors.append(manifest['filename'])
+    for ship in manifest['ships']:
+      s = Ship()
+      row = ship['row']
+      s.manifest_file = m
+      s.bill_type = row[0]
+      s.number = row[1]
+      shipDate = row[2]
+      s.date =  "%s-%s-%s" % (shipDate[6:], shipDate[3:5], shipDate[0:2])
+      s.ship_name = row[3]
+      s.captain = row[4]
+      s.flag = row[5]
+      s.owner = row[6]
+      try:
+        s.port = row[7]
+      except:
+        s.port = ''
+      try:
+        s.country = row[8]
+      except:
+        s.country = ''
+      try:
+        s.save()
+        print "ship success"
+      except:
+        print "ship error"
+        errors_ships.append(row)
+      if s.id:
+        for good in ship['goods']:
+          g = Good()
+          g.ship = s
+          g.description = good[0]
+          g.package_type = good[1]
+          if good[2].strip() != '':
+            try:
+              g.no_of_packages = int(good[2])
+            except:
+              errors_goods.append(good)
+          if good[3].strip() != '':
+            try:
+              g.weight = int(good[3])
+            except:
+              errors_goods.append(good)
+          if good[4].strip() != '':
+            try:
+              g.value = int(good[4])
+            except:
+              errors_goods.append(good)
+          try:
+            g.save()
+          except:
+            print "goods error"
+            errors_goods.append(good)
+  ship_errors_file = open("shipErrors.json", "w")
+  ship_errors_file.write(json.dumps(errors_ships, indent=2))
+  ship_errors_file.close()
+  good_errors_file = open("goodErrors.json", "w")
+  good_errors_file.write(json.dumps(errors_goods, indent=2))
+  good_errors_file.close()
+  manifests_errors_file = open("manifestsErrors.json", "w")
+  manifests_errors_file.write(json.dumps(manifests_errors, indent=2))
+  manifests_errors_file.close()  

manifests/ships/import_data.py

@@ -0,0 +1,8 @@
+import json
+from settings import DATA_PATH
+from os.path import join
+
+json_file_path = join(DATA_PATH, "allData.json")
+
+def importJSON():
+  

manifests/ships/models.py

@@ -0,0 +1,39 @@
+from django.db import models
+
+class Manifest(models.Model):
+  month = models.CharField(max_length=3)
+  year = models.IntegerField(max_length=4)
+  filename = models.CharField(max_length=256)
+
+  def __unicode__(self):
+    return self.filename
+
+#['type', 'number', 'date', 'ship_name', 'captain', 'flag', 'trader', 'port', 'country']
+class Ship(models.Model):
+  manifest_file = models.ForeignKey(Manifest)
+  bill_type = models.CharField(max_length=50)
+  number = models.IntegerField(max_length=20)
+  date = models.DateField()
+  ship_name = models.CharField(max_length=255)
+  captain = models.CharField(max_length=255)
+  flag = models.CharField(max_length=255)
+  owner = models.CharField(max_length=255)
+  port = models.CharField(max_length=255)
+  country = models.CharField(max_length=255)
+
+  def __unicode__(self):
+    return "%d: %s" % (self.number, self.ship_name,)
+
+#['description', 'package_type', 'no_of_packages', 'weight', 'value']
+class Good(models.Model):
+  ship = models.ForeignKey(Ship)
+  description = models.TextField()
+  package_type = models.CharField(max_length=256)
+  no_of_packages = models.IntegerField(max_length=20, null=True)
+  weight = models.IntegerField(max_length=20, null=True)
+  value = models.IntegerField(max_length=20, null=True)
+
+  def __unicode__(self):
+    return self.description
+
+# Create your models here.

manifests/ships/tests.py

@@ -0,0 +1,23 @@
+"""
+This file demonstrates two different styles of tests (one doctest and one
+unittest). These will both pass when you run "manage.py test".
+
+Replace these with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+class SimpleTest(TestCase):
+    def test_basic_addition(self):
+        """
+        Tests that 1 + 1 always equals 2.
+        """
+        self.failUnlessEqual(1 + 1, 2)
+
+__test__ = {"doctest": """
+Another way to test that 1 + 1 is equal to 2.
+
+>>> 1 + 1 == 2
+True
+"""}
+

manifests/ships/views.py

@@ -0,0 +1 @@
+# Create your views here.

manifests/urls.py

@@ -0,0 +1,16 @@
+from django.conf.urls.defaults import *
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+urlpatterns = patterns('',
+    # Example:
+    # (r'^manifests/', include('manifests.foo.urls')),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    (r'^admin/', include(admin.site.urls)),
+)