diff --git a/src/c3nav/mapdata/management/commands/loadmap.py b/src/c3nav/mapdata/management/commands/loadmap.py index e160b2ed..0ce3205c 100644 --- a/src/c3nav/mapdata/management/commands/loadmap.py +++ b/src/c3nav/mapdata/management/commands/loadmap.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from ...packageio import read_packages +from ...packageio import MapdataReader class Command(BaseCommand): @@ -12,8 +12,11 @@ class Command(BaseCommand): help='don\'t ask for confirmation') def handle(self, *args, **options): + reader = MapdataReader() + reader.read_packages() + with transaction.atomic(): - read_packages() + reader.apply_to_db() print() if not options['yes'] and input('Confirm (y/N): ') != 'y': raise CommandError('Aborted.') diff --git a/src/c3nav/mapdata/models/feature.py b/src/c3nav/mapdata/models/feature.py index 938aaaa6..e271d83d 100644 --- a/src/c3nav/mapdata/models/feature.py +++ b/src/c3nav/mapdata/models/feature.py @@ -11,6 +11,7 @@ from ..fields import GeometryField class FeatureType(namedtuple('FeatureType', ('name', 'title', 'title_plural', 'geomtype', 'color'))): + # noinspection PyUnusedLocal def __init__(self, *args, **kwargs): super().__init__() FEATURE_TYPES[self.name] = self @@ -42,6 +43,8 @@ class Feature(models.Model): verbose_name=_('level')) geometry = GeometryField() + path_regex = r'^features/('+'|'.join(name for name, title in TYPES)+')/' + @property def titles(self): return {title.language: title.title for title in self.featuretitles.all()} diff --git a/src/c3nav/mapdata/models/level.py b/src/c3nav/mapdata/models/level.py index 33bf636e..17f7d474 100644 --- a/src/c3nav/mapdata/models/level.py +++ b/src/c3nav/mapdata/models/level.py @@ -1,6 +1,8 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ +from c3nav.mapdata.models import Package + class Level(models.Model): """ @@ -9,20 +11,20 @@ class Level(models.Model): name = models.SlugField(_('level name'), primary_key=True, max_length=50, help_text=_('Usually just an integer (e.g. -1, 0, 1, 2)')) altitude = models.DecimalField(_('level altitude'), null=True, max_digits=6, decimal_places=2) - package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels', + package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='levels', verbose_name=_('map package')) + path_regex = r'^levels/' + @classmethod - def fromfile(cls, data, package, name): + def fromfile(cls, data): if 'altitude' not in data: - raise ValueError('%s.json: missing altitude.' % name) + raise ValueError('missing altitude.') if not isinstance(data['altitude'], (int, float)): - raise ValueError('%s.json: altitude has to be in or float.') + raise ValueError('altitude has to be int or float.') return { - 'name': name, - 'package': package, 'altitude': data['altitude'], } diff --git a/src/c3nav/mapdata/models/package.py b/src/c3nav/mapdata/models/package.py index ae0f3f9b..b626b689 100644 --- a/src/c3nav/mapdata/models/package.py +++ b/src/c3nav/mapdata/models/package.py @@ -22,33 +22,34 @@ class Package(models.Model): directory = models.CharField(_('folder name'), max_length=100) + path_regex = r'^package.json$' + @classmethod - def fromfile(cls, data, directory): - kwargs = { - 'directory': directory - } + def fromfile(cls, data): + kwargs = {} if 'name' not in data: - raise ValueError('pkg.json: missing package name.') + raise ValueError('missing package name.') kwargs['name'] = data['name'] depends = data.get('depends', []) if not isinstance(depends, list): - raise TypeError('pkg.json: depends has to be a list') + raise TypeError('depends has to be a list') kwargs['depends'] = depends - if 'home_repo' in data: - kwargs['home_repo'] = data['home_repo'] + kwargs['home_repo'] = data['home_repo'] if 'home_repo' in data else None if 'bounds' in data: bounds = data['bounds'] if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: - raise ValueError('pkg.json: Invalid bounds format.') + raise ValueError('Invalid bounds format.') if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): - raise ValueError('pkg.json: All bounds coordinates have to be int or float.') + raise ValueError('All bounds coordinates have to be int or float.') if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: - raise ValueError('pkg.json: bounds: lower coordinate has to be first.') - (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds + raise ValueError('bounds: lower coordinate has to be first.') + else: + bounds = (None, None), (None, None) + (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds return kwargs @@ -64,7 +65,7 @@ class Package(models.Model): def bounds(self): if self.bottom is None: return None - return ((float(self.bottom), float(self.left)), (float(self.top), float(self.right))) + return (float(self.bottom), float(self.left)), (float(self.top), float(self.right)) def tofile(self): data = OrderedDict() diff --git a/src/c3nav/mapdata/models/source.py b/src/c3nav/mapdata/models/source.py index ee1aa560..0169ec2a 100644 --- a/src/c3nav/mapdata/models/source.py +++ b/src/c3nav/mapdata/models/source.py @@ -1,13 +1,15 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ +from c3nav.mapdata.models import Package + class Source(models.Model): """ A map source, images of levels that can be useful as backgrounds for the map editor """ name = models.SlugField(_('source name'), primary_key=True, max_length=50) - package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='sources', + package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='sources', verbose_name=_('map package')) bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2) @@ -15,6 +17,8 @@ class Source(models.Model): top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2) right = models.DecimalField(_('right coordinate'), max_digits=6, decimal_places=2) + path_regex = r'^sources/' + @classmethod def max_bounds(cls): result = cls.objects.all().aggregate(models.Min('bottom'), models.Min('left'), @@ -24,25 +28,22 @@ class Source(models.Model): @property def bounds(self): - return ((float(self.bottom), float(self.left)), (float(self.top), float(self.right))) + return (float(self.bottom), float(self.left)), (float(self.top), float(self.right)) @classmethod - def fromfile(cls, data, package, name): - kwargs = { - 'package': package, - 'name': name, - } + def fromfile(cls, data): + kwargs = {} if 'bounds' not in data: - raise ValueError('%s.json: missing bounds.' % name) + raise ValueError('missing bounds.') bounds = data['bounds'] if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: - raise ValueError('pkg.json: Invalid bounds format.') + raise ValueError('Invalid bounds format.') if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): - raise ValueError('pkg.json: All bounds coordinates have to be int or float.') + raise ValueError('All bounds coordinates have to be int or float.') if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: - raise ValueError('pkg.json: bounds: lower coordinate has to be first.') + raise ValueError('bounds: lower coordinate has to be first.') (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds return kwargs diff --git a/src/c3nav/mapdata/packageio/__init__.py b/src/c3nav/mapdata/packageio/__init__.py index b47581b1..b82477d6 100644 --- a/src/c3nav/mapdata/packageio/__init__.py +++ b/src/c3nav/mapdata/packageio/__init__.py @@ -1,3 +1,2 @@ -from .read import read_packages, read_package # noqa +from .read import MapdataReader # noqa from .write import write_packages, write_package # noqa -from .utils import ObjectCollection # noqa diff --git a/src/c3nav/mapdata/packageio/read.py b/src/c3nav/mapdata/packageio/read.py index 59f9cc38..85117177 100644 --- a/src/c3nav/mapdata/packageio/read.py +++ b/src/c3nav/mapdata/packageio/read.py @@ -1,71 +1,163 @@ import json import os +import re import subprocess from django.conf import settings -from django.core.management.base import CommandError +from django.core.management import CommandError -from ..models import Level, Package, Source -from .utils import ObjectCollection +from ..models import Feature, Level, Package, Source -def read_packages(): - print('Detecting Map Packages…') +class MapdataReader: + ordered_models = (Package, Level, Source, Feature) - objects = ObjectCollection() - for directory in os.listdir(settings.MAP_ROOT): - print('\n'+directory) - if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)): - continue - read_package(directory, objects) + def __init__(self): + self.content = {} + self.package_names_by_dir = {} + self.saved_items = {model: {} for model in self.ordered_models} - objects.apply_to_db() + def read_packages(self): + print('Detecting Map Packages…') + + for directory in os.listdir(settings.MAP_ROOT): + print('\n' + directory) + if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)): + continue + self.read_package(directory) + + def read_package(self, package_dir): + full_package_dir = os.path.join(settings.MAP_ROOT, package_dir) + + for path, sub_dirs, filenames in os.walk(full_package_dir): + sub_dirs[:] = sorted([directory for directory in sub_dirs if not directory.startswith('.')]) + for filename in sorted(filenames): + if not filename.endswith('.json'): + continue + self.add_file(package_dir, path[len(full_package_dir) + 1:], filename) + + def _add_item(self, item): + if item.package_dir not in self.content: + self.content[item.package_dir] = {model: [] for model in self.ordered_models} + self.content[item.package_dir][item.model].append(item) + + def add_file(self, package_dir, path, filename): + file_path = os.path.join(package_dir, path, filename) + relative_file_path = os.path.join(path, filename) + print(file_path) + for model in self.ordered_models: + if re.search(model.path_regex, relative_file_path): + self._add_item(ReaderItem(self, package_dir, path, filename, model)) + break + else: + raise CommandError('Unexpected JSON file: %s' % file_path) + + def apply_to_db(self): + # Collect all Packages + package_items_by_name = {} + package_dirs_by_name = {} + for package_dir, items_by_model in self.content.items(): + if not items_by_model[Package]: + raise CommandError('Missing package file: %s' % package_dir) + + if len(items_by_model[Package]) > 1: + raise CommandError('Multiple package files: %s' % package_dir) + + package_item = items_by_model[Package][0] + package_items_by_name[package_item.data['name']] = package_item + package_dirs_by_name[package_item.data['name']] = package_dir + self.package_names_by_dir[package_dir] = package_item.data['name'] + + # Resolve Package Dependencies + unresolved_packages = set(package_items_by_name.keys()) + resolved_packages = set() + package_order = [] + while unresolved_packages: + resolvable = set([name for name in unresolved_packages if + not set(package_items_by_name[name].data['depends'])-resolved_packages]) + if not resolvable: + raise CommandError('Could not resolve package dependencies: %s' % unresolved_packages) + package_order.extend(resolvable) + unresolved_packages -= resolvable + resolved_packages |= resolvable + + # Create new and update existing entries + for package_name in package_order: + print('') + package_dir = package_dirs_by_name[package_name] + items_by_model = self.content[package_dir] + for model in self.ordered_models: + items = items_by_model[model] + for item in items: + item.save() + + # Delete old entries + for model in reversed(self.ordered_models): + queryset = model.objects.exclude(name__in=self.saved_items[model].keys()) + for name in queryset.values_list('name', flat=True): + print('- Deleted %s: %s' % (model.__name__, name)) + queryset.delete() -def read_package(directory, objects=None): - if objects is None: - objects = ObjectCollection() +class ReaderItem: + def __init__(self, reader, package_dir, path, filename, model): + self.reader = reader + self.package_dir = package_dir + self.path = path + self.filename = filename + self.model = model + self.obj = None + self.path_in_package = os.path.join(self.path, self.filename) - path = os.path.join(settings.MAP_ROOT, directory) + try: + with open(os.path.join(settings.MAP_ROOT, package_dir, path, filename)) as f: + self.content = f.read() + except Exception as e: + raise CommandError('Could not read File: %s' % e) - # Main JSON - try: - package = json.load(open(os.path.join(path, 'pkg.json'))) - except FileNotFoundError: - raise CommandError('no pkg.json found') + try: + self.json_data = json.loads(self.content) + except json.JSONDecodeError as e: + raise CommandError('Could not decode JSON: %s' % e) - package = Package.fromfile(package, directory) + self.data = {'name': filename[:-5]} - try: - result = subprocess.Popen(['git', '-C', path, 'rev-parse', '--verify', 'HEAD'], stdout=subprocess.PIPE) - returncode = result.wait() - except: - pass - else: - if returncode == 0: - package['commit_id'] = result.stdout.read().strip() + if self.model == Package: + self.data['commit_id'] = None + try: + full_package_dir = os.path.join(settings.MAP_ROOT, package_dir) + result = subprocess.Popen(['git', '-C', full_package_dir, 'rev-parse', '--verify', 'HEAD'], + stdout=subprocess.PIPE) + returncode = result.wait() + except FileNotFoundError: + pass + else: + if returncode == 0: + self.data['commit_id'] = result.stdout.read().strip() - objects.add_package(package) - objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels'))) - objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True)) - return objects + try: + add_data = self.model.fromfile(self.json_data) + except Exception as e: + raise CommandError('Could not load data: %s' % e) + self.data.update(add_data) + relations = { + 'level': Level, + } -def _read_folder(package, cls, path, check_sister_file=False): - objects = [] - if not os.path.isdir(path): - return [] - for filename in sorted(os.listdir(path)): - if not filename.endswith('.json'): - continue + def save(self): + if self.model != Package: + package_name = self.reader.package_names_by_dir[self.package_dir] + self.data['package'] = self.reader.saved_items[Package][package_name].obj - full_filename = os.path.join(path, filename) - if not os.path.isfile(full_filename): - continue + # Change name references to the referenced object + for name, model in self.relations.items(): + if name in self.data: + self.data[name] = self.reader.saved_items[model][self.data[name]].obj - name = filename[:-5] - if check_sister_file and os.path.isfile(name): - raise CommandError('%s: %s is missing.' % (filename, name)) + obj, created = self.model.objects.update_or_create(name=self.data['name'], defaults=self.data) + if created: + print('- Created %s: %s' % (self.model.__name__, obj.name)) - objects.append(cls.fromfile(json.load(open(full_filename)), package, name)) - return objects + self.obj = obj + self.reader.saved_items[self.model][obj.name] = self diff --git a/src/c3nav/mapdata/packageio/utils.py b/src/c3nav/mapdata/packageio/utils.py deleted file mode 100644 index 56e8095e..00000000 --- a/src/c3nav/mapdata/packageio/utils.py +++ /dev/null @@ -1,98 +0,0 @@ -import json - -from django.core.management.base import CommandError - -from ..models import Level, Package, Source -from ..utils import json_encoder_reindent - - -class ObjectCollection: - def __init__(self): - self.packages = {} - self.levels = {} - self.sources = {} - - def add_package(self, package): - self._add(self.packages, 'package', package) - - def add_level(self, level): - self._add(self.levels, 'level', level) - - def add_source(self, source): - self._add(self.sources, 'source', source) - - def add_packages(self, packages): - for package in packages: - self.add_package(package) - - def add_levels(self, levels): - for level in levels: - self.add_level(level) - - def add_sources(self, sources): - for source in sources: - self.add_source(source) - - def _add(self, container, name, item): - if item['name'] in container: - raise CommandError('Duplicate %s name: %s' % (name, item['name'])) - container[item['name']] = item - - def apply_to_db(self): - for name, package in tuple(self.packages.items()): - for depname in package['depends']: - if depname not in self.packages: - raise CommandError('Missing dependency: %s' % depname) - - for name, package in tuple(self.packages.items()): - package = package.copy() - orig_deps = package.pop('depends', []) - package, created = Package.objects.update_or_create(name=name, defaults=package) - package.orig_deps = orig_deps - self.packages[name] = package - if created: - print('- Created package: '+name) - - for name, level in self.levels.items(): - level['package'] = self.packages[level['package']] - level, created = Level.objects.update_or_create(name=name, defaults=level) - self.levels[name] = level - if created: - print('- Created level: '+name) - - for name, source in self.sources.items(): - source['package'] = self.packages[source['package']] - source, created = Source.objects.update_or_create(name=name, defaults=source) - self.sources[name] = source - if created: - print('- Created source: '+name) - - for source in Source.objects.exclude(name__in=self.sources.keys()): - print('- Deleted source: '+source.name) - source.delete() - - for level in Level.objects.exclude(name__in=self.levels.keys()): - print('- Deleted level: '+level.name) - level.delete() - - for package in Package.objects.exclude(name__in=self.packages.keys()): - print('- Deleted package: '+package.name) - package.delete() - - for name, package in tuple(self.packages.items()): - has_deps = [] - for dependency in tuple(package.depends.all()): - if dependency.name not in package.orig_deps: - package.depends.remove(dependency) - print('- Removed dependency: '+dependency.name) - else: - has_deps.append(dependency.name) - - for depname in package.orig_deps: - if depname not in has_deps: - package.depends.add(self.packages[depname]) - print('- Added dependency: '+depname) - - -def json_encode(data): - return json_encoder_reindent(json.dumps, data, indent=4)+'\n' diff --git a/src/c3nav/mapdata/packageio/write.py b/src/c3nav/mapdata/packageio/write.py index f542d2e3..4cbff1d1 100644 --- a/src/c3nav/mapdata/packageio/write.py +++ b/src/c3nav/mapdata/packageio/write.py @@ -7,8 +7,9 @@ from datetime import datetime from django.conf import settings from django.utils import timezone +from c3nav.mapdata.utils import json_encoder_reindent + from ..models import Package -from .utils import json_encode def write_packages(prettify=False, check_only=False): @@ -109,3 +110,7 @@ def _write_object(obj, path, filename, prettify=False, check_only=False): with open(full_filename, 'w') as f: f.write(new_data_encoded) return 1 + + +def json_encode(data): + return json_encoder_reindent(json.dumps, data, indent=4)+'\n'