diff --git a/src/c3nav/control/admin.py b/src/c3nav/control/admin.py index 8c38f3f3..6af52da2 100644 --- a/src/c3nav/control/admin.py +++ b/src/c3nav/control/admin.py @@ -1,3 +1,3 @@ -from django.contrib import admin +from django.contrib import admin # noqa # Register your models here. diff --git a/src/c3nav/control/models.py b/src/c3nav/control/models.py index 71a83623..9d57c559 100644 --- a/src/c3nav/control/models.py +++ b/src/c3nav/control/models.py @@ -1,3 +1,3 @@ -from django.db import models +from django.db import models # noqa # Create your models here. diff --git a/src/c3nav/control/templates/control/editor.html b/src/c3nav/control/templates/control/editor.html index ba9aec08..bc99f2a6 100644 --- a/src/c3nav/control/templates/control/editor.html +++ b/src/c3nav/control/templates/control/editor.html @@ -19,9 +19,9 @@ var map = L.map('mapeditor', { }); // Add Source Layers -{% for pkg in packages %} +{% for source_list in sources %} L.control.layers([], { - {% for source in pkg.sources.all %} + {% for source in source_list %} "{{ source.name }}": L.imageOverlay('{% url 'map.source' source=source.name %}', {{ source.jsbounds }}),{% endfor %} }).addTo(map); {% endfor %} diff --git a/src/c3nav/control/views.py b/src/c3nav/control/views.py index 85c83df3..5ed0a1a7 100644 --- a/src/c3nav/control/views.py +++ b/src/c3nav/control/views.py @@ -19,7 +19,7 @@ def editor(request, level=None): level = get_object_or_404(Level, name=level) return render(request, 'control/editor.html', { 'bounds': json.dumps(Source.max_bounds()), - 'packages': Package.objects.all(), + 'sources': [p.sources.all().order_by('name') for p in Package.objects.all()], 'levels': Level.objects.all(), 'current_level': level, }) diff --git a/src/c3nav/mapdata/management/commands/loadmappkgs.py b/src/c3nav/mapdata/management/commands/loadmap.py similarity index 66% rename from src/c3nav/mapdata/management/commands/loadmappkgs.py rename to src/c3nav/mapdata/management/commands/loadmap.py index 58336eaa..c31af94f 100644 --- a/src/c3nav/mapdata/management/commands/loadmappkgs.py +++ b/src/c3nav/mapdata/management/commands/loadmap.py @@ -1,19 +1,19 @@ from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from ...packageio import MapPackagesIO + +from ...packageio import read_packages class Command(BaseCommand): - help = 'Load the given map packages into the database' + help = 'Update the map database' def add_arguments(self, parser): - parser.add_argument('mappkgdir', nargs='+', type=str, help='map package directories') parser.add_argument('-y', action='store_const', const=True, default=False, help='don\'t ask for confirmation') def handle(self, *args, **options): with transaction.atomic(): - MapPackagesIO(options['mappkgdir']).update_to_db() + read_packages() print() if input('Confirm (y/N): ') != 'y': raise CommandError('Aborted.') diff --git a/src/c3nav/mapdata/migrations/0001_initial.py b/src/c3nav/mapdata/migrations/0001_initial.py index 8d680c63..82e7dd0e 100644 --- a/src/c3nav/mapdata/migrations/0001_initial.py +++ b/src/c3nav/mapdata/migrations/0001_initial.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.9.9 on 2016-08-29 16:48 +# Generated by Django 1.9.9 on 2016-08-29 20:00 from __future__ import unicode_literals import django.db.models.deletion from django.db import migrations, models -import c3nav.mapdata.models.source import parler.models @@ -39,10 +38,10 @@ class Migration(migrations.Migration): ('master', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='mapdata.Feature')), ], options={ - 'db_tablespace': '', + 'managed': True, 'default_permissions': (), 'db_table': 'mapdata_feature_translation', - 'managed': True, + 'db_tablespace': '', 'verbose_name': 'feature Translation', }, ), @@ -66,6 +65,7 @@ class Migration(migrations.Migration): ('left', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='left coordinate')), ('top', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='top coordinate')), ('right', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='right coordinate')), + ('directory', models.CharField(max_length=100, verbose_name='folder name')), ], ), migrations.CreateModel( @@ -73,7 +73,6 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.SlugField(unique=True, verbose_name='source name')), - ('image', models.FileField(max_length=70, storage=c3nav.mapdata.models.source.SourceImageStorage(), upload_to=c3nav.mapdata.models.source.map_source_filename, verbose_name='source image')), ('bottom', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='bottom coordinate')), ('left', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='left coordinate')), ('top', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='top coordinate')), diff --git a/src/c3nav/mapdata/models/level.py b/src/c3nav/mapdata/models/level.py index 32d807fa..14171590 100644 --- a/src/c3nav/mapdata/models/level.py +++ b/src/c3nav/mapdata/models/level.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -12,5 +14,25 @@ class Level(models.Model): package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels', verbose_name=_('map package')) + def jsonize(self): + return OrderedDict(( + ('name', self.name), + ('altitude', float(self.altitude)), + )) + + @classmethod + def fromfile(cls, data, package, name): + if 'altitude' not in data: + raise ValueError('%s.json: missing altitude.' % name) + + if not isinstance(data['altitude'], (int, float)): + raise ValueError('%s.json: altitude has to be in or float.') + + return { + 'package': package, + 'name': name, + 'altitude': data['altitude'], + } + class Meta: ordering = ['altitude'] diff --git a/src/c3nav/mapdata/models/package.py b/src/c3nav/mapdata/models/package.py index 8378153f..9e3a3f00 100644 --- a/src/c3nav/mapdata/models/package.py +++ b/src/c3nav/mapdata/models/package.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -13,3 +15,35 @@ class Package(models.Model): left = models.DecimalField(_('left coordinate'), null=True, max_digits=6, decimal_places=2) top = models.DecimalField(_('top coordinate'), null=True, max_digits=6, decimal_places=2) right = models.DecimalField(_('right coordinate'), null=True, max_digits=6, decimal_places=2) + + directory = models.CharField(_('folder name'), max_length=100) + + @classmethod + def fromfile(cls, data, directory): + kwargs = { + 'directory': directory + } + + if 'name' not in data: + raise ValueError('pkg.json: missing package name.') + kwargs['name'] = data['name'] + + if 'bounds' in data: + bounds = data['bounds'] + if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: + raise ValueError('pkg.json: Invalid bounds format.') + if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): + raise ValueError('pkg.json: All bounds coordinates have to be int or float.') + if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: + raise ValueError('pkg.json: bounds: lower coordinate has to be first.') + (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds + + return kwargs + + def tofile(self): + data = OrderedDict() + data['name'] = self.name + if self.bottom is not None: + data['bounds'] = ((float(self.bottom), float(self.left)), (float(self.top), float(self.right))) + + return data diff --git a/src/c3nav/mapdata/models/source.py b/src/c3nav/mapdata/models/source.py index bd38fac4..745f4b2d 100644 --- a/src/c3nav/mapdata/models/source.py +++ b/src/c3nav/mapdata/models/source.py @@ -1,24 +1,10 @@ import json -import os +from collections import OrderedDict -from django.conf import settings -from django.core.files.storage import FileSystemStorage -from django.db import models, transaction -from django.dispatch import receiver +from django.db import models from django.utils.translation import ugettext_lazy as _ -class SourceImageStorage(FileSystemStorage): - def get_available_name(self, name, *args, max_length=None, **kwargs): - if self.exists(name): - os.remove(os.path.join(settings.MEDIA_ROOT, name)) - return super().get_available_name(name, *args, max_length, **kwargs) - - -def map_source_filename(instance, filename): - return os.path.join('mapsources', '%s.%s' % (instance.name, filename.split('.')[-1])) - - class Source(models.Model): """ A map source, images of levels that can be useful as backgrounds for the map editor @@ -27,9 +13,6 @@ class Source(models.Model): package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='sources', verbose_name=_('map package')) - image = models.FileField(_('source image'), max_length=70, - upload_to=map_source_filename, storage=SourceImageStorage()) - bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2) left = models.DecimalField(_('left coordinate'), max_digits=6, decimal_places=2) top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2) @@ -50,23 +33,30 @@ class Source(models.Model): def jsbounds(self): return json.dumps(((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))) + @classmethod + def fromfile(cls, data, package, name): + kwargs = { + 'package': package, + 'name': name, + } -@receiver(models.signals.post_delete, sender=Source) -def delete_image_on_mapsource_delete(sender, instance, **kwargs): - transaction.on_commit(lambda: instance.image.delete(save=False)) + if 'bounds' not in data: + raise ValueError('%s.json: missing bounds.' % name) + bounds = data['bounds'] + if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: + raise ValueError('pkg.json: Invalid bounds format.') + if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): + raise ValueError('pkg.json: All bounds coordinates have to be int or float.') + if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: + raise ValueError('pkg.json: bounds: lower coordinate has to be first.') + (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds -@receiver(models.signals.pre_save, sender=Source) -def delete_image_on_mapsource_change(sender, instance, **kwargs): - if not instance.pk: - return False + return kwargs - try: - old_file = Source.objects.get(pk=instance.pk).image - except Source.DoesNotExist: - return False - - new_file = instance.image - - if map_source_filename(instance, new_file.name) != old_file.name: - transaction.on_commit(lambda: old_file.delete(save=False)) + def jsonize(self): + return OrderedDict(( + ('name', self.name), + ('src', 'sources/'+self.get_export_filename()), + ('bounds', ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))), + )) diff --git a/src/c3nav/mapdata/packageio.py b/src/c3nav/mapdata/packageio.py index db5cf79a..23857bf2 100644 --- a/src/c3nav/mapdata/packageio.py +++ b/src/c3nav/mapdata/packageio.py @@ -1,133 +1,157 @@ import json import os -from collections import OrderedDict -from django.core.files import File +from django.conf import settings from django.core.management.base import CommandError from .models import Level, Package, Source -class PackageIOError(CommandError): - pass +class ObjectCollection: + def __init__(self): + self.packages = {} + self.levels = {} + self.sources = {} + def add_package(self, package): + self._add(self.packages, 'package', package) -class MapPackagesIO(): - def __init__(self, directories): - print('Opening Map Packages…') - self.packages = OrderedDict() - self.levels = OrderedDict() - self.sources = OrderedDict() + def add_level(self, level): + self._add(self.levels, 'level', level) - for directory in directories: - print('- '+directory) + def add_source(self, source): + self._add(self.sources, 'source', source) - try: - package = json.load(open(os.path.join(directory, 'pkg.json'))) - except FileNotFoundError: - raise PackageIOError('no pkg.json found in %s' % directory) + def add_packages(self, packages): + for package in packages: + self.add_package(package) - if package['name'] in self.packages: - raise PackageIOError('Duplicate package name: %s' % package['name']) + def add_levels(self, levels): + for level in levels: + self.add_level(level) - if 'bounds' in package: - self._validate_bounds(package['bounds']) + def add_sources(self, sources): + for source in sources: + self.add_source(source) - package['directory'] = directory - self.packages[package['name']] = package + def _add(self, container, name, item): + if item['name'] in container: + raise CommandError('Duplicate %s name: %s' % (name, item['name'])) + container[item['name']] = item - for level in package.get('levels', []): - level = level.copy() - if level['name'] in self.levels: - raise PackageIOError('Duplicate level name: %s in packages %s and %s' % - (level, self.levels[level]['name'], package['name'])) - - if not isinstance(level['altitude'], (int, float)): - raise PackageIOError('levels: %s: altitude has to be int or float.' % level['name']) - - level['package'] = package['name'] - self.levels[level['name']] = level - - for source in package.get('sources', []): - source = source.copy() - if source['name'] in self.sources: - raise PackageIOError('Duplicate source name: %s in packages %s and %s' % - (source['name'], self.sources[source['name']]['name'], package['name'])) - - self._validate_bounds(source['bounds'], 'sources: %s: ' % source['name']) - - source['filename'] = os.path.join(directory, source['src']) - if not os.path.isfile(source['filename']): - raise PackageIOError('Source file not found: '+source['filename']) - - source['package'] = package['name'] - self.sources[source['name']] = source - - def _validate_bounds(self, bounds, prefix=''): - if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: - raise PackageIOError(prefix+'Invalid bounds format.') - if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): - raise PackageIOError(prefix+'All bounds coordinates have to be int or float.') - if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: - raise PackageIOError(prefix+'bounds: lower coordinate has to be first.') - - def update_to_db(self): - from .models import MapPackage, MapLevel, MapSource - print('Updating Map database…') - - # Add new Packages - packages = {} - print('- Updating packages…') - for name, package in self.packages.items(): - bounds = package.get('bounds') - defaults = { - 'bottom': bounds[0][0], - 'left': bounds[0][1], - 'top': bounds[1][0], - 'right': bounds[1][1], - } if bounds else {} - - package, created = MapPackage.objects.update_or_create(name=name, defaults=defaults) - packages[name] = package + def apply_to_db(self): + for name, package in tuple(self.packages.items()): + package, created = Package.objects.update_or_create(name=name, defaults=package) + self.packages[name] = package if created: print('- Created package: '+name) - # Add new levels - print('- Updating levels…') for name, level in self.levels.items(): - package, created = MapLevel.objects.update_or_create(name=name, defaults={ - 'package': packages[level['package']], - 'altitude': level['altitude'], - 'name': level['name'], - }) + level['package'] = self.packages[level['package']] + level, created = Level.objects.update_or_create(name=name, defaults=level) + self.levels[name] = level if created: print('- Created level: '+name) - # Add new map sources - print('- Updating sources…') for name, source in self.sources.items(): - source, created = MapSource.objects.update_or_create(name=name, defaults={ - 'package': packages[source['package']], - 'image': File(open(source['filename'], 'rb')), - 'bottom': source['bounds'][0][0], - 'left': source['bounds'][0][1], - 'top': source['bounds'][1][0], - 'right': source['bounds'][1][1], - }) + source['package'] = self.packages[source['package']] + source, created = Source.objects.update_or_create(name=name, defaults=source) + self.sources[name] = source if created: print('- Created source: '+name) - # Remove old sources - for source in MapSource.objects.exclude(name__in=self.sources.keys()): + for source in Source.objects.exclude(name__in=self.sources.keys()): print('- Deleted source: '+source.name) source.delete() - # Remove old levels - for level in MapLevel.objects.exclude(name__in=self.levels.keys()): + for level in Level.objects.exclude(name__in=self.levels.keys()): print('- Deleted level: '+level.name) level.delete() - # Remove old packages - for package in MapPackage.objects.exclude(name__in=self.packages.keys()): + for package in Package.objects.exclude(name__in=self.packages.keys()): print('- Deleted package: '+package.name) package.delete() + + +def read_packages(): + print('Detecting Map Packages…') + + objects = ObjectCollection() + for directory in os.listdir(settings.MAP_ROOT): + print('\n'+directory) + if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)): + continue + read_package(directory, objects) + + objects.apply_to_db() + + +def read_package(directory, objects=None): + if objects is None: + objects = ObjectCollection() + + path = os.path.join(settings.MAP_ROOT, directory) + + # Main JSON + try: + package = json.load(open(os.path.join(path, 'pkg.json'))) + except FileNotFoundError: + raise CommandError('no pkg.json found') + + package = Package.fromfile(package, directory) + objects.add_package(package) + objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels'))) + objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True)) + return objects + + +def _read_folder(package, cls, path, check_sister_file=False): + objects = [] + if not os.path.isdir(path): + return [] + for filename in os.listdir(path): + if not filename.endswith('.json'): + continue + + full_filename = os.path.join(path, filename) + if not os.path.isfile(full_filename): + continue + + name = filename[:-5] + if check_sister_file and os.path.isfile(name): + raise CommandError('%s: %s is missing.' % (filename, name)) + + objects.append(cls.fromfile(json.load(open(full_filename)), package, name)) + return objects + + +def _fromfile_validate(cls, data, name): + obj = cls.fromfile(json.loads(data), name=name) + formatted_data = json_encode(obj.tofile()) + if data != formatted_data: + raise CommandError('%s.json is not correctly formatted, its contents are:\n---\n' + + data+'\n---\nbut they should be\n---\n'+formatted_data+'\n---') + + +def _json_encode_preencode(data, magic_marker): + if isinstance(data, dict): + data = data.copy() + for name, value in tuple(data.items()): + if name in ('bounds', ): + data[name] = magic_marker+json.dumps(value)+magic_marker + else: + data[name] = _json_encode_preencode(value, magic_marker) + return data + elif isinstance(data, (tuple, list)): + return tuple(_json_encode_preencode(value, magic_marker) for value in data) + else: + return data + + +def json_encode(data): + magic_marker = '***JSON_MAGIC_MARKER***' + test_encode = json.dumps(data) + while magic_marker in test_encode: + magic_marker += '*' + result = json.dumps(_json_encode_preencode(data, magic_marker), indent=4) + return result.replace('"'+magic_marker, '').replace(magic_marker+'"', '')+'\n' diff --git a/src/c3nav/mapdata/views.py b/src/c3nav/mapdata/views.py index ce44d435..3286ca40 100644 --- a/src/c3nav/mapdata/views.py +++ b/src/c3nav/mapdata/views.py @@ -1,8 +1,11 @@ import mimetypes +import os +from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required +from django.core.files import File from django.http import HttpResponse -from django.shortcuts import get_object_or_404, render +from django.shortcuts import get_object_or_404 from .models import Source @@ -10,7 +13,8 @@ from .models import Source @staff_member_required def source(request, source): source = get_object_or_404(Source, name=source) - response = HttpResponse(content_type=mimetypes.guess_type(source.image.name)[0]) - for chunk in source.image.chunks(): + response = HttpResponse(content_type=mimetypes.guess_type(source.name)[0]) + image_path = os.path.join(settings.MAP_ROOT, source.package.directory, 'sources', source.name) + for chunk in File(open(image_path, 'rb')).chunks(): response.write(chunk) return response diff --git a/src/c3nav/settings.py b/src/c3nav/settings.py index 1fa576c7..241fd4c6 100644 --- a/src/c3nav/settings.py +++ b/src/c3nav/settings.py @@ -17,6 +17,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATA_DIR = config.get('c3nav', 'datadir', fallback=os.environ.get('DATA_DIR', 'data')) LOG_DIR = os.path.join(DATA_DIR, 'logs') MEDIA_ROOT = os.path.join(DATA_DIR, 'media') +MAP_ROOT = os.path.join(DATA_DIR, 'map') if not os.path.exists(DATA_DIR): os.mkdir(DATA_DIR) @@ -24,6 +25,8 @@ if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) if not os.path.exists(MEDIA_ROOT): os.mkdir(MEDIA_ROOT) +if not os.path.exists(MAP_ROOT): + os.mkdir(MAP_ROOT) if config.has_option('django', 'secret'): SECRET_KEY = config.get('django', 'secret')