new db, translations, etc

manifests/settings.py

@@ -5,6 +5,7 @@ from os.path import join
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 JSON_DEBUG = DEBUG
+LOCAL_DEVELOPMENT = True
 
 ADMINS = (
     # ('Your Name', 'your_email@domain.com'),
@@ -26,6 +27,7 @@ DATABASES = {
     }
 }
 
+
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.
@@ -51,13 +53,14 @@ USE_L10N = True
 
 # Absolute path to the directory that holds media.
 # Example: "/home/media/media.lawrence.com/"
-MEDIA_ROOT = join(PROJECT_PATH, "static") 
+MEDIA_ROOT = join(PROJECT_PATH, "media") 
 
+STATIC_ROOT = join(PROJECT_PATH, "static")
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
 # trailing slash if there is a path component (optional in other cases).
 # Examples: "http://media.lawrence.com", "http://example.com/media/"
-MEDIA_URL = '/static/'
+MEDIA_URL = '/media/'
-
+STATIC_URL = '/static/'
 # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
 # trailing slash.
 # Examples: "http://foo.com/media/", "/media/".
@@ -85,13 +88,13 @@ except NameError:
 TEMPLATE_LOADERS = (
     'django.template.loaders.filesystem.Loader',
     'django.template.loaders.app_directories.Loader',
-     'django.template.loaders.eggs.Loader',
+    'django.template.loaders.eggs.Loader',
 )
 
 MIDDLEWARE_CLASSES = (
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
+#    'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
 )
@@ -111,6 +114,7 @@ INSTALLED_APPS = (
     'django.contrib.sessions',
     'django.contrib.sites',
     'django.contrib.messages',
+    'django.contrib.staticfiles',
     'ships',
     # Uncomment the next line to enable the admin:
     'django.contrib.admin',

manifests/ships/admin.py

@@ -10,13 +10,21 @@ class GoodInline(admin.TabularInline):
 
 class ShipAdmin(admin.ModelAdmin):
     inlines = [GoodInline]
+    search_fields = ['ship_name', 'ship_name_trans', 'port']
     list_filter = ('bill_type', 'manifest_file',)
     list_display = ('__unicode__', 'bill_type', 'number', 'date', 'flag', 'captain', 'owner', 'port', 'country', 'no_of_goods',)
 
 class GoodAdmin(admin.ModelAdmin):
-    pass
+    search_fields = ['description', 'description_trans']
+
+
+class TranslationAdmin(admin.ModelAdmin):
+    search_fields = ['string', 'string_trans']
+    list_editable = ('string_trans',)
+    list_display = ('string', 'string_trans',)
 
 
 admin.site.register(Manifest, ManifestAdmin)
 admin.site.register(Good, GoodAdmin)
 admin.site.register(Ship, ShipAdmin)
+admin.site.register(Translation, TranslationAdmin)

manifests/ships/convert.py

@@ -4,6 +4,8 @@ import sys
 import os
 import csv
 import json
+from models import Ship, Good, Manifest, Translation
+import re
 
 def toCsv():
   manifests = glob('*.txt')
@@ -119,7 +121,6 @@ def addType(inFile, outFile):
   outFile.close()
 
 def importJSON(filename):
-  from models import Ship, Good, Manifest
   data = json.loads(open(filename).read())
   errors_ships = []
   errors_goods = []
@@ -197,4 +198,50 @@ def importJSON(filename):
   good_errors_file.close()
   manifests_errors_file = open("manifestsErrors.json", "w")
   manifests_errors_file.write(json.dumps(manifests_errors, indent=2))
-  manifests_errors_file.close()  
+  manifests_errors_file.close()
+
+
+def generateGoodStrings():
+    for g in Good.objects.all():
+        s = getGoodString(g.description)
+        print s
+        g.description_string = s
+        g.save()
+    return
+
+def getGoodString(s):
+    noRegex = re.compile(r'^[0-9]*?$')
+    parts = s.split("/")
+    if len(parts) == 1:
+        return s
+    if len(parts) == 2:
+        if noRegex.match(parts[0].strip()) is not None:
+            return parts[1]
+        elif noRegex.match(parts[1].strip()) is not None:
+            return parts[0]
+        else:
+            return s
+    else:            
+        return s
+                        
+def tokenizeGoods():
+    for g in Good.objects.all():
+        description = g.description
+        parts = description.split("/")
+        for p in parts:
+#            print p
+            w = p.strip()
+            if not isNumerical(w):
+                if Translation.objects.filter(string=w).count() < 1:
+                    t = Translation(string=w)
+                    t.save()
+                    print w
+#        for p in parts:
+            
+
+regex = re.compile(r'.*?[0-9].*')
+def isNumerical(s):
+    if regex.match(s) is not None:
+        return True
+    else:
+        return False

manifests/ships/models.py

@@ -43,6 +43,8 @@ class Ship(models.Model):
 class Good(models.Model):
   ship = models.ForeignKey(Ship)
   description = models.TextField()
+  description_string = models.TextField(blank=True, null=True)
+  description_string_trans = models.TextField(blank=True, null=True)
   package_type = models.CharField(max_length=256)
   no_of_packages = models.IntegerField(max_length=20, null=True)
   weight = models.IntegerField(max_length=20, null=True)
@@ -51,4 +53,15 @@ class Good(models.Model):
   def __unicode__(self):
     return self.description
 
+
+class Translation(models.Model):
+    string = models.TextField(unique=True)
+    string_trans = models.TextField(blank=True)
+    looked_at = models.BooleanField(default=False)
+
+    def __unicode__(self):
+        return "%s: %s" % (self.string, self.string_trans,)
+
+    class Meta:
+        ordering = ('id',)
 # Create your models here.

manifests/ships/views.py

@@ -6,8 +6,11 @@ import json
 from django.shortcuts import render_to_response
 from django.db.models import Q
 from django.template import RequestContext
+from ox.django.shortcuts import render_to_json_response
+from django.views.decorators.csrf import csrf_exempt
 
-def translate(request, model, field):
+
+def googletranslate(request, model, field):
     start = request.GET.get("start", False)
     end = request.GET.get("end", False)
     if not start and not end:
@@ -40,16 +43,106 @@ def translate(request, model, field):
             ordArr.append(str(ord(char)))
         if not isEnglish:
             response.append({
-                'ord': ordArr.join(","),
+                'ord': ",".join(ordArr),
-		'val': val
+		        'val': val
             })
     context = RequestContext(request, {
         'values': response,
         'model': model,
-        'field': field
+        'field': field,
+        'start': start,
+        'end': end
     })
     return render_to_response("translate.html", context)
 
+@csrf_exempt
+def dotranslate(request, model, field):
+    data = json.loads(request.POST.get('data', '{}'))
+    m = models.__getattribute__(model)
+    trans_field = "%s_trans" % field
+    savedObjects = 0
+    for d in data:
+        newVal = d['val']
+        oldVal = "".join([unichr(int(v)) for v in d['ord']])        
+        q = Q(**{"%s__exact" % field: oldVal})
+        matches = m.objects.filter(q)
+        for match in matches:
+            match.__setattr__(trans_field, newVal)
+            match.save()
+            savedObjects += 1
+    return render_to_json_response({'len': savedObjects})
+     
+#    return render_to_json_response(data)
+
+
+
+def translate(request, model, field):
+    start = request.GET.get("start", False)
+    end = request.GET.get("end", False)
+    if not start and not end:
+        limit = False
+    else:
+        limit = True
+    try:
+        m = models.__getattribute__(model)
+    except AttributeError:
+        return HttpResponse("no such model")
+    response = []
+    if field not in m._meta.get_all_field_names():
+        return HttpResponse("no such field")
+    trans_field = field + "_trans"
+    qset = m.objects.values('id', field, trans_field).order_by(field).distinct(field)
+    if limit:
+        if not start:
+            start = 0
+        if not end:
+            end = qset.count()
+        qset = qset[start:end]
+    for obj in qset:
+        response.append({
+            'id': obj['id'],
+            'string': obj[field],
+            'translation': obj[trans_field]
+        })
+    context = RequestContext(request, {
+        'values': response,
+        'model': model,
+        'field': field,
+        'start': start,
+        'end': end
+    })
+    return render_to_response("trans.html", context)
+
+
+def stringtranslate(request):
+    model = request.POST.get("model", "")
+    field = request.POST.get("field", "")
+    id = request.POST.get("id", 0)
+    string = request.POST.get("string", "")
+    translation = request.POST.get("translation", "")
+    if not model == 'Translation':
+        return render_to_json_response({'error': 'this only works for the translation model'})
+    obj = Translation.objects.get(pk=id)
+    obj.translation = translation
+    obj.save()
+    return render_to_json_response({'ok': 'ok'})
+
+'''
+def dotrans(request, model, field):
+    oldVal = request.POST.get('oldVal', '')
+    newVal = request.POST.get('newVal', '')
+    m = models.__getattribute__(model)
+    trans_field = "%s_trans" % field
+    savedObjects = 0
+    q = Q(**{"%s__exact" % trans_field: oldVal})
+    matches = m.objects.filter(q)
+    for match in matches:
+        match.__setattr__(trans_field, newVal)
+        match.save()
+        savedObjects += 1
+    return render_to_json_response({'len': savedObjects})
+
+
 def dotranslate(request, model, field):
     txt = request.POST.get('txt', '')
     lines = txt.split("\n\n")
@@ -68,3 +161,4 @@ def dotranslate(request, model, field):
             m[trans_field] = val
             m.save()
     return HttpResponse(savedArr.join("\n"))
+'''

manifests/templates/translate.html

@@ -1,7 +1,8 @@
 <!doctype html>
 <html>
     <head>
-
+        <script type="text/javascript" src="/media/js/jquery.js"></script>
+        <script type="text/javascript" src="/media/js/dotranslate.js"></script>
     </head>
     <body>
 <div id="google_translate_element"></div><script>
@@ -11,16 +12,26 @@ function googleTranslateElementInit() {
     includedLanguages: 'en'
   }, 'google_translate_element');
 }
+
+var model = "{{ model }}";
+var field = "{{ field }}";
+var start = {{ start }};
+var end = {{ end }};
 </script><script src="//translate.google.com/translate_a/element.js?cb=googleTranslateElementInit"></script>
-	<table>
+    <button id="doTranslate">Save Translations</button>
+    <br />
+	<table id="transTable">
 		{% for v in values %}
 		<tr>
-            <td>{{ v.ord }}</td>
+            <td data-ord="{{ v.ord }}">{{ v.val }}</td>
-            <td>{{ v.val }}</td>
+            <!-- <td>{{ v.ord }}</td> -->
 		</tr>
 		{% endfor %}
 	</table>
     <br /><br />
+    <div id="controlValue">
+        اطارات
+    </div>
         <form action="/dotranslate/{{ model }}/{{ field }}/" method="POST">
             {% csrf_token %}
             <textarea name="txt" id="txt"></textarea> <br />

manifests/urls.py

@@ -1,5 +1,6 @@
 from django.conf.urls.defaults import *
-
+import settings
+from os.path import join
 # Uncomment the next two lines to enable the admin:
 from django.contrib import admin
 admin.autodiscover()
@@ -12,7 +13,24 @@ urlpatterns = patterns('',
     # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
     # Uncomment the next line to enable the admin:
-    (r'translate/(?P<model>[A-Za-z].*?)/(?P<field>[A-Za-z].*?)/', 'ships.views.translate'),
+
-    (r'dotranslate/(?P<model>[A-Za-z].*?)/(?P<field>[A-Za-z].*?)/', 'ships.views.dotranslate'),
+    # for human translators
+    (r'^trans/(?P<model>[A-Za-z].*?)/(?P<field>[A-Za-z].*?)/', 'ships.views.translate'),
+#    (r'^dotrans/(?P<model>[A-Za-z].*?)/(?P<field>[A-Za-z].*?)/', 'ships.views.dotranslate'),
+
+    # for automated google translation
+    (r'^googletranslate/(?P<model>[A-Za-z].*?)/(?P<field>[A-Za-z].*?)/', 'ships.views.googletranslate'),
+    (r'^dotranslate/(?P<model>[A-Za-z].*?)/(?P<field>[A-Za-z].*?)/', 'ships.views.dotranslate'),
+    (r'^stringtranslate/$', 'ships.views.stringtranslate'),
     (r'^admin/', include(admin.site.urls)),
 )
+
+if settings.LOCAL_DEVELOPMENT:
+#
+  urlpatterns += patterns('',
+#
+  (r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': join(settings.PROJECT_PATH, "static")}),
+  (r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': join(settings.PROJECT_PATH, "media")}),
+#
+)
+

requirements.txt

@@ -1,3 +1,3 @@
--e svn+http://code.djangoproject.com/svn/django/branches/releases/1.3.X/#egg=django
+-e svn+http://code.djangoproject.com/svn/django/trunk/#egg=django
 -e bzr+http://code.0x2620.org/python-ox/#egg=python-ox
-django_extensions
+-e git://github.com/bit/django-extensions.git#egg=django_extensions