diff --git a/src/c3nav/mapdata/management/commands/dumpmap.py b/src/c3nav/mapdata/management/commands/dumpmap.py new file mode 100644 index 00000000..b08510ba --- /dev/null +++ b/src/c3nav/mapdata/management/commands/dumpmap.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand +from django.db import transaction + +from ...packageio import write_packages + + +class Command(BaseCommand): + help = 'Dump the map database' + + def add_arguments(self, parser): + parser.add_argument('--no-prettify', dest='prettify', action='store_const', const=False, default=True, + help='dont\'t prettify existing files') + + def handle(self, *args, **options): + with transaction.atomic(): + write_packages(prettify=options['prettify']) + print() diff --git a/src/c3nav/mapdata/migrations/0002_auto_20160830_1028.py b/src/c3nav/mapdata/migrations/0002_auto_20160830_1028.py new file mode 100644 index 00000000..2d7b2ba3 --- /dev/null +++ b/src/c3nav/mapdata/migrations/0002_auto_20160830_1028.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.9 on 2016-08-30 10:28 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mapdata', '0001_initial'), + ] + + operations = [ + migrations.AlterModelOptions( + name='level', + options={}, + ), + ] diff --git a/src/c3nav/mapdata/models/level.py b/src/c3nav/mapdata/models/level.py index 14171590..a791c58a 100644 --- a/src/c3nav/mapdata/models/level.py +++ b/src/c3nav/mapdata/models/level.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -14,12 +12,6 @@ class Level(models.Model): package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels', verbose_name=_('map package')) - def jsonize(self): - return OrderedDict(( - ('name', self.name), - ('altitude', float(self.altitude)), - )) - @classmethod def fromfile(cls, data, package, name): if 'altitude' not in data: @@ -34,5 +26,7 @@ class Level(models.Model): 'altitude': data['altitude'], } - class Meta: - ordering = ['altitude'] + def tofile(self): + return { + 'altitude': float(self.altitude) + } diff --git a/src/c3nav/mapdata/models/package.py b/src/c3nav/mapdata/models/package.py index 9e3a3f00..168780e8 100644 --- a/src/c3nav/mapdata/models/package.py +++ b/src/c3nav/mapdata/models/package.py @@ -40,6 +40,10 @@ class Package(models.Model): return kwargs + @property + def package(self): + return self + def tofile(self): data = OrderedDict() data['name'] = self.name diff --git a/src/c3nav/mapdata/models/source.py b/src/c3nav/mapdata/models/source.py index 745f4b2d..f471bbdf 100644 --- a/src/c3nav/mapdata/models/source.py +++ b/src/c3nav/mapdata/models/source.py @@ -1,5 +1,4 @@ import json -from collections import OrderedDict from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -54,9 +53,7 @@ class Source(models.Model): return kwargs - def jsonize(self): - return OrderedDict(( - ('name', self.name), - ('src', 'sources/'+self.get_export_filename()), - ('bounds', ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))), - )) + def tofile(self): + return { + 'bounds': ((float(self.bottom), float(self.left)), (float(self.top), float(self.right))) + } diff --git a/src/c3nav/mapdata/packageio/__init__.py b/src/c3nav/mapdata/packageio/__init__.py new file mode 100644 index 00000000..b47581b1 --- /dev/null +++ b/src/c3nav/mapdata/packageio/__init__.py @@ -0,0 +1,3 @@ +from .read import read_packages, read_package # noqa +from .write import write_packages, write_package # noqa +from .utils import ObjectCollection # noqa diff --git a/src/c3nav/mapdata/packageio/read.py b/src/c3nav/mapdata/packageio/read.py new file mode 100644 index 00000000..2951cd09 --- /dev/null +++ b/src/c3nav/mapdata/packageio/read.py @@ -0,0 +1,68 @@ +import json +import os + +from django.conf import settings +from django.core.management.base import CommandError + +from ..models import Level, Package, Source +from .utils import ObjectCollection, json_encode + + +def read_packages(): + print('Detecting Map Packages…') + + objects = ObjectCollection() + for directory in os.listdir(settings.MAP_ROOT): + print('\n'+directory) + if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)): + continue + read_package(directory, objects) + + objects.apply_to_db() + + +def read_package(directory, objects=None): + if objects is None: + objects = ObjectCollection() + + path = os.path.join(settings.MAP_ROOT, directory) + + # Main JSON + try: + package = json.load(open(os.path.join(path, 'pkg.json'))) + except FileNotFoundError: + raise CommandError('no pkg.json found') + + package = Package.fromfile(package, directory) + objects.add_package(package) + objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels'))) + objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True)) + return objects + + +def _read_folder(package, cls, path, check_sister_file=False): + objects = [] + if not os.path.isdir(path): + return [] + for filename in os.listdir(path): + if not filename.endswith('.json'): + continue + + full_filename = os.path.join(path, filename) + if not os.path.isfile(full_filename): + continue + + name = filename[:-5] + if check_sister_file and os.path.isfile(name): + raise CommandError('%s: %s is missing.' % (filename, name)) + + objects.append(cls.fromfile(json.load(open(full_filename)), package, name)) + return objects + + +def _fromfile_validate(cls, data, name): + obj = cls.fromfile(json.loads(data), name=name) + formatted_data = json_encode(obj.tofile()) + if data != formatted_data: + raise CommandError('%s.json is not correctly formatted, its contents are:\n---\n' + + data+'\n---\nbut they should be\n---\n'+formatted_data+'\n---') diff --git a/src/c3nav/mapdata/packageio.py b/src/c3nav/mapdata/packageio/utils.py similarity index 55% rename from src/c3nav/mapdata/packageio.py rename to src/c3nav/mapdata/packageio/utils.py index 23857bf2..c39cd0f5 100644 --- a/src/c3nav/mapdata/packageio.py +++ b/src/c3nav/mapdata/packageio/utils.py @@ -1,10 +1,8 @@ import json -import os -from django.conf import settings from django.core.management.base import CommandError -from .models import Level, Package, Source +from ..models import Level, Package, Source class ObjectCollection: @@ -73,77 +71,17 @@ class ObjectCollection: package.delete() -def read_packages(): - print('Detecting Map Packages…') - - objects = ObjectCollection() - for directory in os.listdir(settings.MAP_ROOT): - print('\n'+directory) - if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)): - continue - read_package(directory, objects) - - objects.apply_to_db() - - -def read_package(directory, objects=None): - if objects is None: - objects = ObjectCollection() - - path = os.path.join(settings.MAP_ROOT, directory) - - # Main JSON - try: - package = json.load(open(os.path.join(path, 'pkg.json'))) - except FileNotFoundError: - raise CommandError('no pkg.json found') - - package = Package.fromfile(package, directory) - objects.add_package(package) - objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels'))) - objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True)) - return objects - - -def _read_folder(package, cls, path, check_sister_file=False): - objects = [] - if not os.path.isdir(path): - return [] - for filename in os.listdir(path): - if not filename.endswith('.json'): - continue - - full_filename = os.path.join(path, filename) - if not os.path.isfile(full_filename): - continue - - name = filename[:-5] - if check_sister_file and os.path.isfile(name): - raise CommandError('%s: %s is missing.' % (filename, name)) - - objects.append(cls.fromfile(json.load(open(full_filename)), package, name)) - return objects - - -def _fromfile_validate(cls, data, name): - obj = cls.fromfile(json.loads(data), name=name) - formatted_data = json_encode(obj.tofile()) - if data != formatted_data: - raise CommandError('%s.json is not correctly formatted, its contents are:\n---\n' + - data+'\n---\nbut they should be\n---\n'+formatted_data+'\n---') - - -def _json_encode_preencode(data, magic_marker): +def _preencode(data, magic_marker): if isinstance(data, dict): data = data.copy() for name, value in tuple(data.items()): if name in ('bounds', ): data[name] = magic_marker+json.dumps(value)+magic_marker else: - data[name] = _json_encode_preencode(value, magic_marker) + data[name] = _preencode(value, magic_marker) return data elif isinstance(data, (tuple, list)): - return tuple(_json_encode_preencode(value, magic_marker) for value in data) + return tuple(_preencode(value, magic_marker) for value in data) else: return data @@ -153,5 +91,5 @@ def json_encode(data): test_encode = json.dumps(data) while magic_marker in test_encode: magic_marker += '*' - result = json.dumps(_json_encode_preencode(data, magic_marker), indent=4) + result = json.dumps(_preencode(data, magic_marker), indent=4) return result.replace('"'+magic_marker, '').replace(magic_marker+'"', '')+'\n' diff --git a/src/c3nav/mapdata/packageio/write.py b/src/c3nav/mapdata/packageio/write.py new file mode 100644 index 00000000..a4b95735 --- /dev/null +++ b/src/c3nav/mapdata/packageio/write.py @@ -0,0 +1,63 @@ +import json +import os + +from django.conf import settings + +from ..models import Package +from .utils import json_encode + + +def write_packages(prettify=False): + print('Writing Map Packages…') + for package in Package.objects.all(): + print('\n'+package.name) + write_package(package, prettify) + + +def write_package(package, prettify=False): + pkg_path = os.path.join(settings.MAP_ROOT, package.directory) + + with open(os.path.join(pkg_path, 'pkg.json'), 'w') as f: + f.write(json_encode(package.tofile())) + + _write_folder(package, package.levels.all(), 'levels', prettify) + _write_folder(package, package.sources.all(), 'sources', prettify, check_sister_file=True) + + +def _write_folder(package, objects, path, prettify=False, check_sister_file=False): + filenames = set() + full_path = os.path.join(settings.MAP_ROOT, package.directory, path) + if objects: + if not os.path.isdir(full_path): + os.mkdir(full_path) + for obj in objects: + filename = '%s.json' % obj.name + filenames.add(filename) + + full_filename = os.path.join(full_path, filename) + new_data = obj.tofile() + new_data_encoded = json_encode(new_data) + if os.path.isfile(full_filename): + with open(full_filename) as f: + old_data_encoded = f.read() + old_data = json.loads(old_data_encoded, parse_int=float) + if old_data != json.loads(new_data_encoded, parse_int=float): + print('- Updated: '+os.path.join(path, filename)) + elif old_data_encoded != new_data_encoded: + if not prettify: + continue + print('- Beautified: '+os.path.join(path, filename)) + else: + continue + else: + print('- Created: '+os.path.join(path, filename)) + with open(full_filename, 'w') as f: + f.write(new_data_encoded) + + if os.path.isdir(path): + for filename in os.listdir(path): + full_filename = os.path.join(path, filename) + if filename not in filenames and filename.endswith('.json') and os.path.isfile(full_filename): + os.remove(full_filename) + if check_sister_file and os.path.isfile(full_filename[:-5]): + os.remove(full_filename[:-5])