From ad3d877fc0c153b486ac1f96dd3113cc6fd0efef Mon Sep 17 00:00:00 2001 From: Sanj Date: Sat, 18 Feb 2012 13:27:36 +0530 Subject: [PATCH] new db, translations, etc --- manifests/settings.py | 14 ++-- manifests/ships/admin.py | 10 ++- manifests/ships/convert.py | 51 ++++++++++++++- manifests/ships/models.py | 13 ++++ manifests/ships/views.py | 102 +++++++++++++++++++++++++++-- manifests/templates/translate.html | 19 ++++-- manifests/urls.py | 24 ++++++- requirements.txt | 4 +- 8 files changed, 216 insertions(+), 21 deletions(-) diff --git a/manifests/settings.py b/manifests/settings.py index 0ec12ed..287eb5f 100644 --- a/manifests/settings.py +++ b/manifests/settings.py @@ -5,6 +5,7 @@ from os.path import join DEBUG = True TEMPLATE_DEBUG = DEBUG JSON_DEBUG = DEBUG +LOCAL_DEVELOPMENT = True ADMINS = ( # ('Your Name', 'your_email@domain.com'), @@ -26,6 +27,7 @@ DATABASES = { } } + # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. @@ -51,13 +53,14 @@ USE_L10N = True # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = join(PROJECT_PATH, "static") +MEDIA_ROOT = join(PROJECT_PATH, "media") +STATIC_ROOT = join(PROJECT_PATH, "static") # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" -MEDIA_URL = '/static/' - +MEDIA_URL = '/media/' +STATIC_URL = '/static/' # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a # trailing slash. # Examples: "http://foo.com/media/", "/media/". @@ -85,13 +88,13 @@ except NameError: TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', - 'django.template.loaders.eggs.Loader', + 'django.template.loaders.eggs.Loader', ) MIDDLEWARE_CLASSES = ( 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', +# 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', ) @@ -111,6 +114,7 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.messages', + 'django.contrib.staticfiles', 'ships', # Uncomment the next line to enable the admin: 'django.contrib.admin', diff --git a/manifests/ships/admin.py b/manifests/ships/admin.py index ea033a8..49b2c50 100644 --- a/manifests/ships/admin.py +++ b/manifests/ships/admin.py @@ -10,13 +10,21 @@ class GoodInline(admin.TabularInline): class ShipAdmin(admin.ModelAdmin): inlines = [GoodInline] + search_fields = ['ship_name', 'ship_name_trans', 'port'] list_filter = ('bill_type', 'manifest_file',) list_display = ('__unicode__', 'bill_type', 'number', 'date', 'flag', 'captain', 'owner', 'port', 'country', 'no_of_goods',) class GoodAdmin(admin.ModelAdmin): - pass + search_fields = ['description', 'description_trans'] + + +class TranslationAdmin(admin.ModelAdmin): + search_fields = ['string', 'string_trans'] + list_editable = ('string_trans',) + list_display = ('string', 'string_trans',) admin.site.register(Manifest, ManifestAdmin) admin.site.register(Good, GoodAdmin) admin.site.register(Ship, ShipAdmin) +admin.site.register(Translation, TranslationAdmin) diff --git a/manifests/ships/convert.py b/manifests/ships/convert.py index 8bdb082..e730303 100755 --- a/manifests/ships/convert.py +++ b/manifests/ships/convert.py @@ -4,6 +4,8 @@ import sys import os import csv import json +from models import Ship, Good, Manifest, Translation +import re def toCsv(): manifests = glob('*.txt') @@ -119,7 +121,6 @@ def addType(inFile, outFile): outFile.close() def importJSON(filename): - from models import Ship, Good, Manifest data = json.loads(open(filename).read()) errors_ships = [] errors_goods = [] @@ -197,4 +198,50 @@ def importJSON(filename): good_errors_file.close() manifests_errors_file = open("manifestsErrors.json", "w") manifests_errors_file.write(json.dumps(manifests_errors, indent=2)) - manifests_errors_file.close() + manifests_errors_file.close() + + +def generateGoodStrings(): + for g in Good.objects.all(): + s = getGoodString(g.description) + print s + g.description_string = s + g.save() + return + +def getGoodString(s): + noRegex = re.compile(r'^[0-9]*?$') + parts = s.split("/") + if len(parts) == 1: + return s + if len(parts) == 2: + if noRegex.match(parts[0].strip()) is not None: + return parts[1] + elif noRegex.match(parts[1].strip()) is not None: + return parts[0] + else: + return s + else: + return s + +def tokenizeGoods(): + for g in Good.objects.all(): + description = g.description + parts = description.split("/") + for p in parts: +# print p + w = p.strip() + if not isNumerical(w): + if Translation.objects.filter(string=w).count() < 1: + t = Translation(string=w) + t.save() + print w +# for p in parts: + + +regex = re.compile(r'.*?[0-9].*') +def isNumerical(s): + if regex.match(s) is not None: + return True + else: + return False diff --git a/manifests/ships/models.py b/manifests/ships/models.py index f989804..f3eec93 100644 --- a/manifests/ships/models.py +++ b/manifests/ships/models.py @@ -43,6 +43,8 @@ class Ship(models.Model): class Good(models.Model): ship = models.ForeignKey(Ship) description = models.TextField() + description_string = models.TextField(blank=True, null=True) + description_string_trans = models.TextField(blank=True, null=True) package_type = models.CharField(max_length=256) no_of_packages = models.IntegerField(max_length=20, null=True) weight = models.IntegerField(max_length=20, null=True) @@ -51,4 +53,15 @@ class Good(models.Model): def __unicode__(self): return self.description + +class Translation(models.Model): + string = models.TextField(unique=True) + string_trans = models.TextField(blank=True) + looked_at = models.BooleanField(default=False) + + def __unicode__(self): + return "%s: %s" % (self.string, self.string_trans,) + + class Meta: + ordering = ('id',) # Create your models here. diff --git a/manifests/ships/views.py b/manifests/ships/views.py index d5a44ff..ed41eb8 100644 --- a/manifests/ships/views.py +++ b/manifests/ships/views.py @@ -6,8 +6,11 @@ import json from django.shortcuts import render_to_response from django.db.models import Q from django.template import RequestContext +from ox.django.shortcuts import render_to_json_response +from django.views.decorators.csrf import csrf_exempt -def translate(request, model, field): + +def googletranslate(request, model, field): start = request.GET.get("start", False) end = request.GET.get("end", False) if not start and not end: @@ -40,16 +43,106 @@ def translate(request, model, field): ordArr.append(str(ord(char))) if not isEnglish: response.append({ - 'ord': ordArr.join(","), - 'val': val + 'ord': ",".join(ordArr), + 'val': val }) context = RequestContext(request, { 'values': response, 'model': model, - 'field': field + 'field': field, + 'start': start, + 'end': end }) return render_to_response("translate.html", context) +@csrf_exempt +def dotranslate(request, model, field): + data = json.loads(request.POST.get('data', '{}')) + m = models.__getattribute__(model) + trans_field = "%s_trans" % field + savedObjects = 0 + for d in data: + newVal = d['val'] + oldVal = "".join([unichr(int(v)) for v in d['ord']]) + q = Q(**{"%s__exact" % field: oldVal}) + matches = m.objects.filter(q) + for match in matches: + match.__setattr__(trans_field, newVal) + match.save() + savedObjects += 1 + return render_to_json_response({'len': savedObjects}) + +# return render_to_json_response(data) + + + +def translate(request, model, field): + start = request.GET.get("start", False) + end = request.GET.get("end", False) + if not start and not end: + limit = False + else: + limit = True + try: + m = models.__getattribute__(model) + except AttributeError: + return HttpResponse("no such model") + response = [] + if field not in m._meta.get_all_field_names(): + return HttpResponse("no such field") + trans_field = field + "_trans" + qset = m.objects.values('id', field, trans_field).order_by(field).distinct(field) + if limit: + if not start: + start = 0 + if not end: + end = qset.count() + qset = qset[start:end] + for obj in qset: + response.append({ + 'id': obj['id'], + 'string': obj[field], + 'translation': obj[trans_field] + }) + context = RequestContext(request, { + 'values': response, + 'model': model, + 'field': field, + 'start': start, + 'end': end + }) + return render_to_response("trans.html", context) + + +def stringtranslate(request): + model = request.POST.get("model", "") + field = request.POST.get("field", "") + id = request.POST.get("id", 0) + string = request.POST.get("string", "") + translation = request.POST.get("translation", "") + if not model == 'Translation': + return render_to_json_response({'error': 'this only works for the translation model'}) + obj = Translation.objects.get(pk=id) + obj.translation = translation + obj.save() + return render_to_json_response({'ok': 'ok'}) + +''' +def dotrans(request, model, field): + oldVal = request.POST.get('oldVal', '') + newVal = request.POST.get('newVal', '') + m = models.__getattribute__(model) + trans_field = "%s_trans" % field + savedObjects = 0 + q = Q(**{"%s__exact" % trans_field: oldVal}) + matches = m.objects.filter(q) + for match in matches: + match.__setattr__(trans_field, newVal) + match.save() + savedObjects += 1 + return render_to_json_response({'len': savedObjects}) + + def dotranslate(request, model, field): txt = request.POST.get('txt', '') lines = txt.split("\n\n") @@ -68,3 +161,4 @@ def dotranslate(request, model, field): m[trans_field] = val m.save() return HttpResponse(savedArr.join("\n")) +''' diff --git a/manifests/templates/translate.html b/manifests/templates/translate.html index 7510716..6d5a399 100644 --- a/manifests/templates/translate.html +++ b/manifests/templates/translate.html @@ -1,7 +1,8 @@ - + +
- + +
+
{% for v in values %} - - + + {% endfor %}
{{ v.ord }}{{ v.val }}{{ v.val }}


+
+ اطارات +
{% csrf_token %}
diff --git a/manifests/urls.py b/manifests/urls.py index 6fcd15f..8b8a6a7 100644 --- a/manifests/urls.py +++ b/manifests/urls.py @@ -1,5 +1,6 @@ from django.conf.urls.defaults import * - +import settings +from os.path import join # Uncomment the next two lines to enable the admin: from django.contrib import admin admin.autodiscover() @@ -12,7 +13,24 @@ urlpatterns = patterns('', # (r'^admin/doc/', include('django.contrib.admindocs.urls')), # Uncomment the next line to enable the admin: - (r'translate/(?P[A-Za-z].*?)/(?P[A-Za-z].*?)/', 'ships.views.translate'), - (r'dotranslate/(?P[A-Za-z].*?)/(?P[A-Za-z].*?)/', 'ships.views.dotranslate'), + + # for human translators + (r'^trans/(?P[A-Za-z].*?)/(?P[A-Za-z].*?)/', 'ships.views.translate'), +# (r'^dotrans/(?P[A-Za-z].*?)/(?P[A-Za-z].*?)/', 'ships.views.dotranslate'), + + # for automated google translation + (r'^googletranslate/(?P[A-Za-z].*?)/(?P[A-Za-z].*?)/', 'ships.views.googletranslate'), + (r'^dotranslate/(?P[A-Za-z].*?)/(?P[A-Za-z].*?)/', 'ships.views.dotranslate'), + (r'^stringtranslate/$', 'ships.views.stringtranslate'), (r'^admin/', include(admin.site.urls)), ) + +if settings.LOCAL_DEVELOPMENT: +# + urlpatterns += patterns('', +# + (r'^static/(?P.*)$', 'django.views.static.serve', {'document_root': join(settings.PROJECT_PATH, "static")}), + (r'^media/(?P.*)$', 'django.views.static.serve', {'document_root': join(settings.PROJECT_PATH, "media")}), +# +) + diff --git a/requirements.txt b/requirements.txt index 23673db..44f01bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ --e svn+http://code.djangoproject.com/svn/django/branches/releases/1.3.X/#egg=django +-e svn+http://code.djangoproject.com/svn/django/trunk/#egg=django -e bzr+http://code.0x2620.org/python-ox/#egg=python-ox -django_extensions +-e git://github.com/bit/django-extensions.git#egg=django_extensions