refactor models, change map package storage and map package loading

src/c3nav/control/admin.py

@@ -1,3 +1,3 @@
-from django.contrib import admin
+from django.contrib import admin  # noqa
 
 # Register your models here.

src/c3nav/control/models.py

@@ -1,3 +1,3 @@
-from django.db import models
+from django.db import models  # noqa
 
 # Create your models here.

src/c3nav/control/templates/control/editor.html

@@ -19,9 +19,9 @@ var map = L.map('mapeditor', {
 });
 
 // Add Source Layers
-{% for pkg in packages %}
+{% for source_list in sources %}
 L.control.layers([], {
-    {% for source in pkg.sources.all %}
+    {% for source in source_list %}
     "{{ source.name }}": L.imageOverlay('{% url 'map.source' source=source.name %}', {{ source.jsbounds }}),{% endfor %}
 }).addTo(map);
 {% endfor %}

src/c3nav/control/views.py

@@ -19,7 +19,7 @@ def editor(request, level=None):
     level = get_object_or_404(Level, name=level)
     return render(request, 'control/editor.html', {
         'bounds': json.dumps(Source.max_bounds()),
-        'packages': Package.objects.all(),
+        'sources': [p.sources.all().order_by('name') for p in Package.objects.all()],
         'levels': Level.objects.all(),
         'current_level': level,
     })

src/c3nav/mapdata/management/commands/loadmap.py

@@ -1,19 +1,19 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
-from ...packageio import MapPackagesIO
+
+from ...packageio import read_packages
 
 
 class Command(BaseCommand):
-    help = 'Load the given map packages into the database'
+    help = 'Update the map database'
 
     def add_arguments(self, parser):
-        parser.add_argument('mappkgdir', nargs='+', type=str, help='map package directories')
         parser.add_argument('-y', action='store_const', const=True, default=False,
                             help='don\'t ask for confirmation')
 
     def handle(self, *args, **options):
         with transaction.atomic():
-            MapPackagesIO(options['mappkgdir']).update_to_db()
+            read_packages()
             print()
             if input('Confirm (y/N): ') != 'y':
                 raise CommandError('Aborted.')

src/c3nav/mapdata/migrations/0001_initial.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.9.9 on 2016-08-29 16:48
+# Generated by Django 1.9.9 on 2016-08-29 20:00
 from __future__ import unicode_literals
 
 import django.db.models.deletion
 from django.db import migrations, models
 
-import c3nav.mapdata.models.source
 import parler.models
 
 
@@ -39,10 +38,10 @@ class Migration(migrations.Migration):
                 ('master', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='mapdata.Feature')),
             ],
             options={
-                'db_tablespace': '',
+                'managed': True,
                 'default_permissions': (),
                 'db_table': 'mapdata_feature_translation',
-                'managed': True,
+                'db_tablespace': '',
                 'verbose_name': 'feature Translation',
             },
         ),
@@ -66,6 +65,7 @@ class Migration(migrations.Migration):
                 ('left', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='left coordinate')),
                 ('top', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='top coordinate')),
                 ('right', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='right coordinate')),
+                ('directory', models.CharField(max_length=100, verbose_name='folder name')),
             ],
         ),
         migrations.CreateModel(
@@ -73,7 +73,6 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.SlugField(unique=True, verbose_name='source name')),
-                ('image', models.FileField(max_length=70, storage=c3nav.mapdata.models.source.SourceImageStorage(), upload_to=c3nav.mapdata.models.source.map_source_filename, verbose_name='source image')),
                 ('bottom', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='bottom coordinate')),
                 ('left', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='left coordinate')),
                 ('top', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='top coordinate')),

src/c3nav/mapdata/models/level.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
@@ -12,5 +14,25 @@ class Level(models.Model):
     package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels',
                                 verbose_name=_('map package'))
 
+    def jsonize(self):
+        return OrderedDict((
+            ('name', self.name),
+            ('altitude', float(self.altitude)),
+        ))
+
+    @classmethod
+    def fromfile(cls, data, package, name):
+        if 'altitude' not in data:
+            raise ValueError('%s.json: missing altitude.' % name)
+
+        if not isinstance(data['altitude'], (int, float)):
+            raise ValueError('%s.json: altitude has to be in or float.')
+
+        return {
+            'package': package,
+            'name': name,
+            'altitude': data['altitude'],
+        }
+
     class Meta:
         ordering = ['altitude']

src/c3nav/mapdata/models/package.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
@@ -13,3 +15,35 @@ class Package(models.Model):
     left = models.DecimalField(_('left coordinate'), null=True, max_digits=6, decimal_places=2)
     top = models.DecimalField(_('top coordinate'), null=True, max_digits=6, decimal_places=2)
     right = models.DecimalField(_('right coordinate'), null=True, max_digits=6, decimal_places=2)
+
+    directory = models.CharField(_('folder name'), max_length=100)
+
+    @classmethod
+    def fromfile(cls, data, directory):
+        kwargs = {
+            'directory': directory
+        }
+
+        if 'name' not in data:
+            raise ValueError('pkg.json: missing package name.')
+        kwargs['name'] = data['name']
+
+        if 'bounds' in data:
+            bounds = data['bounds']
+            if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
+                raise ValueError('pkg.json: Invalid bounds format.')
+            if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
+                raise ValueError('pkg.json: All bounds coordinates have to be int or float.')
+            if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
+                raise ValueError('pkg.json: bounds: lower coordinate has to be first.')
+            (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
+
+        return kwargs
+
+    def tofile(self):
+        data = OrderedDict()
+        data['name'] = self.name
+        if self.bottom is not None:
+            data['bounds'] = ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))
+
+        return data

src/c3nav/mapdata/models/source.py

@@ -1,24 +1,10 @@
 import json
-import os
+from collections import OrderedDict
 
-from django.conf import settings
+from django.db import models
-from django.core.files.storage import FileSystemStorage
-from django.db import models, transaction
-from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 
 
-class SourceImageStorage(FileSystemStorage):
-    def get_available_name(self, name, *args, max_length=None, **kwargs):
-        if self.exists(name):
-            os.remove(os.path.join(settings.MEDIA_ROOT, name))
-        return super().get_available_name(name, *args, max_length, **kwargs)
-
-
-def map_source_filename(instance, filename):
-    return os.path.join('mapsources', '%s.%s' % (instance.name, filename.split('.')[-1]))
-
-
 class Source(models.Model):
     """
     A map source, images of levels that can be useful as backgrounds for the map editor
@@ -27,9 +13,6 @@ class Source(models.Model):
     package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='sources',
                                 verbose_name=_('map package'))
 
-    image = models.FileField(_('source image'), max_length=70,
-                             upload_to=map_source_filename, storage=SourceImageStorage())
-
     bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2)
     left = models.DecimalField(_('left coordinate'), max_digits=6, decimal_places=2)
     top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2)
@@ -50,23 +33,30 @@ class Source(models.Model):
     def jsbounds(self):
         return json.dumps(((float(self.bottom), float(self.left)), (float(self.top), float(self.right))))
 
+    @classmethod
+    def fromfile(cls, data, package, name):
+        kwargs = {
+            'package': package,
+            'name': name,
+        }
 
-@receiver(models.signals.post_delete, sender=Source)
+        if 'bounds' not in data:
-def delete_image_on_mapsource_delete(sender, instance, **kwargs):
+            raise ValueError('%s.json: missing bounds.' % name)
-    transaction.on_commit(lambda: instance.image.delete(save=False))
 
+        bounds = data['bounds']
+        if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
+            raise ValueError('pkg.json: Invalid bounds format.')
+        if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
+            raise ValueError('pkg.json: All bounds coordinates have to be int or float.')
+        if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
+            raise ValueError('pkg.json: bounds: lower coordinate has to be first.')
+        (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
 
-@receiver(models.signals.pre_save, sender=Source)
+        return kwargs
-def delete_image_on_mapsource_change(sender, instance, **kwargs):
-    if not instance.pk:
-        return False
 
-    try:
+    def jsonize(self):
-        old_file = Source.objects.get(pk=instance.pk).image
+        return OrderedDict((
-    except Source.DoesNotExist:
+            ('name', self.name),
-        return False
+            ('src', 'sources/'+self.get_export_filename()),
-
+            ('bounds', ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))),
-    new_file = instance.image
+        ))
-
-    if map_source_filename(instance, new_file.name) != old_file.name:
-        transaction.on_commit(lambda: old_file.delete(save=False))

src/c3nav/mapdata/packageio.py

@@ -1,133 +1,157 @@
 import json
 import os
-from collections import OrderedDict
 
-from django.core.files import File
+from django.conf import settings
 from django.core.management.base import CommandError
 
 from .models import Level, Package, Source
 
 
-class PackageIOError(CommandError):
+class ObjectCollection:
-    pass
+    def __init__(self):
+        self.packages = {}
+        self.levels = {}
+        self.sources = {}
 
+    def add_package(self, package):
+        self._add(self.packages, 'package', package)
 
-class MapPackagesIO():
+    def add_level(self, level):
-    def __init__(self, directories):
+        self._add(self.levels, 'level', level)
-        print('Opening Map Packages…')
-        self.packages = OrderedDict()
-        self.levels = OrderedDict()
-        self.sources = OrderedDict()
 
-        for directory in directories:
+    def add_source(self, source):
-            print('- '+directory)
+        self._add(self.sources, 'source', source)
 
-            try:
+    def add_packages(self, packages):
-                package = json.load(open(os.path.join(directory, 'pkg.json')))
+        for package in packages:
-            except FileNotFoundError:
+            self.add_package(package)
-                raise PackageIOError('no pkg.json found in %s' % directory)
 
-            if package['name'] in self.packages:
+    def add_levels(self, levels):
-                raise PackageIOError('Duplicate package name: %s' % package['name'])
+        for level in levels:
+            self.add_level(level)
 
-            if 'bounds' in package:
+    def add_sources(self, sources):
-                self._validate_bounds(package['bounds'])
+        for source in sources:
+            self.add_source(source)
 
-            package['directory'] = directory
+    def _add(self, container, name, item):
-            self.packages[package['name']] = package
+        if item['name'] in container:
+            raise CommandError('Duplicate %s name: %s' % (name, item['name']))
+        container[item['name']] = item
 
-            for level in package.get('levels', []):
+    def apply_to_db(self):
-                level = level.copy()
+        for name, package in tuple(self.packages.items()):
-                if level['name'] in self.levels:
+            package, created = Package.objects.update_or_create(name=name, defaults=package)
-                    raise PackageIOError('Duplicate level name: %s in packages %s and %s' %
+            self.packages[name] = package
-                                         (level, self.levels[level]['name'], package['name']))
-
-                if not isinstance(level['altitude'], (int, float)):
-                    raise PackageIOError('levels: %s: altitude has to be int or float.' % level['name'])
-
-                level['package'] = package['name']
-                self.levels[level['name']] = level
-
-            for source in package.get('sources', []):
-                source = source.copy()
-                if source['name'] in self.sources:
-                    raise PackageIOError('Duplicate source name: %s in packages %s and %s' %
-                                         (source['name'], self.sources[source['name']]['name'], package['name']))
-
-                self._validate_bounds(source['bounds'], 'sources: %s: ' % source['name'])
-
-                source['filename'] = os.path.join(directory, source['src'])
-                if not os.path.isfile(source['filename']):
-                    raise PackageIOError('Source file not found: '+source['filename'])
-
-                source['package'] = package['name']
-                self.sources[source['name']] = source
-
-    def _validate_bounds(self, bounds, prefix=''):
-        if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
-            raise PackageIOError(prefix+'Invalid bounds format.')
-        if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
-            raise PackageIOError(prefix+'All bounds coordinates have to be int or float.')
-        if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
-            raise PackageIOError(prefix+'bounds: lower coordinate has to be first.')
-
-    def update_to_db(self):
-        from .models import MapPackage, MapLevel, MapSource
-        print('Updating Map database…')
-
-        # Add new Packages
-        packages = {}
-        print('- Updating packages…')
-        for name, package in self.packages.items():
-            bounds = package.get('bounds')
-            defaults = {
-                'bottom': bounds[0][0],
-                'left': bounds[0][1],
-                'top': bounds[1][0],
-                'right': bounds[1][1],
-            } if bounds else {}
-
-            package, created = MapPackage.objects.update_or_create(name=name, defaults=defaults)
-            packages[name] = package
             if created:
                 print('- Created package: '+name)
 
-        # Add new levels
-        print('- Updating levels…')
         for name, level in self.levels.items():
-            package, created = MapLevel.objects.update_or_create(name=name, defaults={
+            level['package'] = self.packages[level['package']]
-                'package': packages[level['package']],
+            level, created = Level.objects.update_or_create(name=name, defaults=level)
-                'altitude': level['altitude'],
+            self.levels[name] = level
-                'name': level['name'],
-            })
             if created:
                 print('- Created level: '+name)
 
-        # Add new map sources
-        print('- Updating sources…')
         for name, source in self.sources.items():
-            source, created = MapSource.objects.update_or_create(name=name, defaults={
+            source['package'] = self.packages[source['package']]
-                'package': packages[source['package']],
+            source, created = Source.objects.update_or_create(name=name, defaults=source)
-                'image': File(open(source['filename'], 'rb')),
+            self.sources[name] = source
-                'bottom': source['bounds'][0][0],
-                'left': source['bounds'][0][1],
-                'top': source['bounds'][1][0],
-                'right': source['bounds'][1][1],
-            })
             if created:
                 print('- Created source: '+name)
 
-        # Remove old sources
+        for source in Source.objects.exclude(name__in=self.sources.keys()):
-        for source in MapSource.objects.exclude(name__in=self.sources.keys()):
             print('- Deleted source: '+source.name)
             source.delete()
 
-        # Remove old levels
+        for level in Level.objects.exclude(name__in=self.levels.keys()):
-        for level in MapLevel.objects.exclude(name__in=self.levels.keys()):
             print('- Deleted level: '+level.name)
             level.delete()
 
-        # Remove old packages
+        for package in Package.objects.exclude(name__in=self.packages.keys()):
-        for package in MapPackage.objects.exclude(name__in=self.packages.keys()):
             print('- Deleted package: '+package.name)
             package.delete()
+
+
+def read_packages():
+    print('Detecting Map Packages…')
+
+    objects = ObjectCollection()
+    for directory in os.listdir(settings.MAP_ROOT):
+        print('\n'+directory)
+        if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)):
+            continue
+        read_package(directory, objects)
+
+    objects.apply_to_db()
+
+
+def read_package(directory, objects=None):
+    if objects is None:
+        objects = ObjectCollection()
+
+    path = os.path.join(settings.MAP_ROOT, directory)
+
+    # Main JSON
+    try:
+        package = json.load(open(os.path.join(path, 'pkg.json')))
+    except FileNotFoundError:
+        raise CommandError('no pkg.json found')
+
+    package = Package.fromfile(package, directory)
+    objects.add_package(package)
+    objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels')))
+    objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True))
+    return objects
+
+
+def _read_folder(package, cls, path, check_sister_file=False):
+    objects = []
+    if not os.path.isdir(path):
+        return []
+    for filename in os.listdir(path):
+        if not filename.endswith('.json'):
+            continue
+
+        full_filename = os.path.join(path, filename)
+        if not os.path.isfile(full_filename):
+            continue
+
+        name = filename[:-5]
+        if check_sister_file and os.path.isfile(name):
+            raise CommandError('%s: %s is missing.' % (filename, name))
+
+        objects.append(cls.fromfile(json.load(open(full_filename)), package, name))
+    return objects
+
+
+def _fromfile_validate(cls, data, name):
+    obj = cls.fromfile(json.loads(data), name=name)
+    formatted_data = json_encode(obj.tofile())
+    if data != formatted_data:
+        raise CommandError('%s.json is not correctly formatted, its contents are:\n---\n' +
+                           data+'\n---\nbut they should be\n---\n'+formatted_data+'\n---')
+
+
+def _json_encode_preencode(data, magic_marker):
+    if isinstance(data, dict):
+        data = data.copy()
+        for name, value in tuple(data.items()):
+            if name in ('bounds', ):
+                data[name] = magic_marker+json.dumps(value)+magic_marker
+            else:
+                data[name] = _json_encode_preencode(value, magic_marker)
+        return data
+    elif isinstance(data, (tuple, list)):
+        return tuple(_json_encode_preencode(value, magic_marker) for value in data)
+    else:
+        return data
+
+
+def json_encode(data):
+    magic_marker = '***JSON_MAGIC_MARKER***'
+    test_encode = json.dumps(data)
+    while magic_marker in test_encode:
+        magic_marker += '*'
+    result = json.dumps(_json_encode_preencode(data, magic_marker), indent=4)
+    return result.replace('"'+magic_marker, '').replace(magic_marker+'"', '')+'\n'

src/c3nav/mapdata/views.py

@@ -1,8 +1,11 @@
 import mimetypes
+import os
 
+from django.conf import settings
 from django.contrib.admin.views.decorators import staff_member_required
+from django.core.files import File
 from django.http import HttpResponse
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import get_object_or_404
 
 from .models import Source
 
@@ -10,7 +13,8 @@ from .models import Source
 @staff_member_required
 def source(request, source):
     source = get_object_or_404(Source, name=source)
-    response = HttpResponse(content_type=mimetypes.guess_type(source.image.name)[0])
+    response = HttpResponse(content_type=mimetypes.guess_type(source.name)[0])
-    for chunk in source.image.chunks():
+    image_path = os.path.join(settings.MAP_ROOT, source.package.directory, 'sources', source.name)
+    for chunk in File(open(image_path, 'rb')).chunks():
         response.write(chunk)
     return response

src/c3nav/settings.py

@@ -17,6 +17,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 DATA_DIR = config.get('c3nav', 'datadir', fallback=os.environ.get('DATA_DIR', 'data'))
 LOG_DIR = os.path.join(DATA_DIR, 'logs')
 MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
+MAP_ROOT = os.path.join(DATA_DIR, 'map')
 
 if not os.path.exists(DATA_DIR):
     os.mkdir(DATA_DIR)
@@ -24,6 +25,8 @@ if not os.path.exists(LOG_DIR):
     os.mkdir(LOG_DIR)
 if not os.path.exists(MEDIA_ROOT):
     os.mkdir(MEDIA_ROOT)
+if not os.path.exists(MAP_ROOT):
+    os.mkdir(MAP_ROOT)
 
 if config.has_option('django', 'secret'):
     SECRET_KEY = config.get('django', 'secret')