rewrite loadmap implementation

This commit is contained in:
Laura Klünder 2016-09-24 14:09:52 +02:00
parent e074f70799
commit 995ccffc67
9 changed files with 190 additions and 182 deletions

View file

@ -1,7 +1,7 @@
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from django.db import transaction from django.db import transaction
from ...packageio import read_packages from ...packageio import MapdataReader
class Command(BaseCommand): class Command(BaseCommand):
@ -12,8 +12,11 @@ class Command(BaseCommand):
help='don\'t ask for confirmation') help='don\'t ask for confirmation')
def handle(self, *args, **options): def handle(self, *args, **options):
reader = MapdataReader()
reader.read_packages()
with transaction.atomic(): with transaction.atomic():
read_packages() reader.apply_to_db()
print() print()
if not options['yes'] and input('Confirm (y/N): ') != 'y': if not options['yes'] and input('Confirm (y/N): ') != 'y':
raise CommandError('Aborted.') raise CommandError('Aborted.')

View file

@ -11,6 +11,7 @@ from ..fields import GeometryField
class FeatureType(namedtuple('FeatureType', ('name', 'title', 'title_plural', 'geomtype', 'color'))): class FeatureType(namedtuple('FeatureType', ('name', 'title', 'title_plural', 'geomtype', 'color'))):
# noinspection PyUnusedLocal
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__() super().__init__()
FEATURE_TYPES[self.name] = self FEATURE_TYPES[self.name] = self
@ -42,6 +43,8 @@ class Feature(models.Model):
verbose_name=_('level')) verbose_name=_('level'))
geometry = GeometryField() geometry = GeometryField()
path_regex = r'^features/('+'|'.join(name for name, title in TYPES)+')/'
@property @property
def titles(self): def titles(self):
return {title.language: title.title for title in self.featuretitles.all()} return {title.language: title.title for title in self.featuretitles.all()}

View file

@ -1,6 +1,8 @@
from django.db import models from django.db import models
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.models import Package
class Level(models.Model): class Level(models.Model):
""" """
@ -9,20 +11,20 @@ class Level(models.Model):
name = models.SlugField(_('level name'), primary_key=True, max_length=50, name = models.SlugField(_('level name'), primary_key=True, max_length=50,
help_text=_('Usually just an integer (e.g. -1, 0, 1, 2)')) help_text=_('Usually just an integer (e.g. -1, 0, 1, 2)'))
altitude = models.DecimalField(_('level altitude'), null=True, max_digits=6, decimal_places=2) altitude = models.DecimalField(_('level altitude'), null=True, max_digits=6, decimal_places=2)
package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels', package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='levels',
verbose_name=_('map package')) verbose_name=_('map package'))
path_regex = r'^levels/'
@classmethod @classmethod
def fromfile(cls, data, package, name): def fromfile(cls, data):
if 'altitude' not in data: if 'altitude' not in data:
raise ValueError('%s.json: missing altitude.' % name) raise ValueError('missing altitude.')
if not isinstance(data['altitude'], (int, float)): if not isinstance(data['altitude'], (int, float)):
raise ValueError('%s.json: altitude has to be in or float.') raise ValueError('altitude has to be int or float.')
return { return {
'name': name,
'package': package,
'altitude': data['altitude'], 'altitude': data['altitude'],
} }

View file

@ -22,33 +22,34 @@ class Package(models.Model):
directory = models.CharField(_('folder name'), max_length=100) directory = models.CharField(_('folder name'), max_length=100)
path_regex = r'^package.json$'
@classmethod @classmethod
def fromfile(cls, data, directory): def fromfile(cls, data):
kwargs = { kwargs = {}
'directory': directory
}
if 'name' not in data: if 'name' not in data:
raise ValueError('pkg.json: missing package name.') raise ValueError('missing package name.')
kwargs['name'] = data['name'] kwargs['name'] = data['name']
depends = data.get('depends', []) depends = data.get('depends', [])
if not isinstance(depends, list): if not isinstance(depends, list):
raise TypeError('pkg.json: depends has to be a list') raise TypeError('depends has to be a list')
kwargs['depends'] = depends kwargs['depends'] = depends
if 'home_repo' in data: kwargs['home_repo'] = data['home_repo'] if 'home_repo' in data else None
kwargs['home_repo'] = data['home_repo']
if 'bounds' in data: if 'bounds' in data:
bounds = data['bounds'] bounds = data['bounds']
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise ValueError('pkg.json: Invalid bounds format.') raise ValueError('Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise ValueError('pkg.json: All bounds coordinates have to be int or float.') raise ValueError('All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise ValueError('pkg.json: bounds: lower coordinate has to be first.') raise ValueError('bounds: lower coordinate has to be first.')
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds else:
bounds = (None, None), (None, None)
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
return kwargs return kwargs
@ -64,7 +65,7 @@ class Package(models.Model):
def bounds(self): def bounds(self):
if self.bottom is None: if self.bottom is None:
return None return None
return ((float(self.bottom), float(self.left)), (float(self.top), float(self.right))) return (float(self.bottom), float(self.left)), (float(self.top), float(self.right))
def tofile(self): def tofile(self):
data = OrderedDict() data = OrderedDict()

View file

@ -1,13 +1,15 @@
from django.db import models from django.db import models
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.models import Package
class Source(models.Model): class Source(models.Model):
""" """
A map source, images of levels that can be useful as backgrounds for the map editor A map source, images of levels that can be useful as backgrounds for the map editor
""" """
name = models.SlugField(_('source name'), primary_key=True, max_length=50) name = models.SlugField(_('source name'), primary_key=True, max_length=50)
package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='sources', package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='sources',
verbose_name=_('map package')) verbose_name=_('map package'))
bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2) bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2)
@ -15,6 +17,8 @@ class Source(models.Model):
top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2) top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2)
right = models.DecimalField(_('right coordinate'), max_digits=6, decimal_places=2) right = models.DecimalField(_('right coordinate'), max_digits=6, decimal_places=2)
path_regex = r'^sources/'
@classmethod @classmethod
def max_bounds(cls): def max_bounds(cls):
result = cls.objects.all().aggregate(models.Min('bottom'), models.Min('left'), result = cls.objects.all().aggregate(models.Min('bottom'), models.Min('left'),
@ -24,25 +28,22 @@ class Source(models.Model):
@property @property
def bounds(self): def bounds(self):
return ((float(self.bottom), float(self.left)), (float(self.top), float(self.right))) return (float(self.bottom), float(self.left)), (float(self.top), float(self.right))
@classmethod @classmethod
def fromfile(cls, data, package, name): def fromfile(cls, data):
kwargs = { kwargs = {}
'package': package,
'name': name,
}
if 'bounds' not in data: if 'bounds' not in data:
raise ValueError('%s.json: missing bounds.' % name) raise ValueError('missing bounds.')
bounds = data['bounds'] bounds = data['bounds']
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2: if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise ValueError('pkg.json: Invalid bounds format.') raise ValueError('Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])): if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise ValueError('pkg.json: All bounds coordinates have to be int or float.') raise ValueError('All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]: if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise ValueError('pkg.json: bounds: lower coordinate has to be first.') raise ValueError('bounds: lower coordinate has to be first.')
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds (kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
return kwargs return kwargs

View file

@ -1,3 +1,2 @@
from .read import read_packages, read_package # noqa from .read import MapdataReader # noqa
from .write import write_packages, write_package # noqa from .write import write_packages, write_package # noqa
from .utils import ObjectCollection # noqa

View file

@ -1,71 +1,163 @@
import json import json
import os import os
import re
import subprocess import subprocess
from django.conf import settings from django.conf import settings
from django.core.management.base import CommandError from django.core.management import CommandError
from ..models import Level, Package, Source from ..models import Feature, Level, Package, Source
from .utils import ObjectCollection
def read_packages(): class MapdataReader:
print('Detecting Map Packages…') ordered_models = (Package, Level, Source, Feature)
objects = ObjectCollection() def __init__(self):
for directory in os.listdir(settings.MAP_ROOT): self.content = {}
print('\n'+directory) self.package_names_by_dir = {}
if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)): self.saved_items = {model: {} for model in self.ordered_models}
continue
read_package(directory, objects)
objects.apply_to_db() def read_packages(self):
print('Detecting Map Packages…')
for directory in os.listdir(settings.MAP_ROOT):
print('\n' + directory)
if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)):
continue
self.read_package(directory)
def read_package(self, package_dir):
full_package_dir = os.path.join(settings.MAP_ROOT, package_dir)
for path, sub_dirs, filenames in os.walk(full_package_dir):
sub_dirs[:] = sorted([directory for directory in sub_dirs if not directory.startswith('.')])
for filename in sorted(filenames):
if not filename.endswith('.json'):
continue
self.add_file(package_dir, path[len(full_package_dir) + 1:], filename)
def _add_item(self, item):
if item.package_dir not in self.content:
self.content[item.package_dir] = {model: [] for model in self.ordered_models}
self.content[item.package_dir][item.model].append(item)
def add_file(self, package_dir, path, filename):
file_path = os.path.join(package_dir, path, filename)
relative_file_path = os.path.join(path, filename)
print(file_path)
for model in self.ordered_models:
if re.search(model.path_regex, relative_file_path):
self._add_item(ReaderItem(self, package_dir, path, filename, model))
break
else:
raise CommandError('Unexpected JSON file: %s' % file_path)
def apply_to_db(self):
# Collect all Packages
package_items_by_name = {}
package_dirs_by_name = {}
for package_dir, items_by_model in self.content.items():
if not items_by_model[Package]:
raise CommandError('Missing package file: %s' % package_dir)
if len(items_by_model[Package]) > 1:
raise CommandError('Multiple package files: %s' % package_dir)
package_item = items_by_model[Package][0]
package_items_by_name[package_item.data['name']] = package_item
package_dirs_by_name[package_item.data['name']] = package_dir
self.package_names_by_dir[package_dir] = package_item.data['name']
# Resolve Package Dependencies
unresolved_packages = set(package_items_by_name.keys())
resolved_packages = set()
package_order = []
while unresolved_packages:
resolvable = set([name for name in unresolved_packages if
not set(package_items_by_name[name].data['depends'])-resolved_packages])
if not resolvable:
raise CommandError('Could not resolve package dependencies: %s' % unresolved_packages)
package_order.extend(resolvable)
unresolved_packages -= resolvable
resolved_packages |= resolvable
# Create new and update existing entries
for package_name in package_order:
print('')
package_dir = package_dirs_by_name[package_name]
items_by_model = self.content[package_dir]
for model in self.ordered_models:
items = items_by_model[model]
for item in items:
item.save()
# Delete old entries
for model in reversed(self.ordered_models):
queryset = model.objects.exclude(name__in=self.saved_items[model].keys())
for name in queryset.values_list('name', flat=True):
print('- Deleted %s: %s' % (model.__name__, name))
queryset.delete()
def read_package(directory, objects=None): class ReaderItem:
if objects is None: def __init__(self, reader, package_dir, path, filename, model):
objects = ObjectCollection() self.reader = reader
self.package_dir = package_dir
self.path = path
self.filename = filename
self.model = model
self.obj = None
self.path_in_package = os.path.join(self.path, self.filename)
path = os.path.join(settings.MAP_ROOT, directory) try:
with open(os.path.join(settings.MAP_ROOT, package_dir, path, filename)) as f:
self.content = f.read()
except Exception as e:
raise CommandError('Could not read File: %s' % e)
# Main JSON try:
try: self.json_data = json.loads(self.content)
package = json.load(open(os.path.join(path, 'pkg.json'))) except json.JSONDecodeError as e:
except FileNotFoundError: raise CommandError('Could not decode JSON: %s' % e)
raise CommandError('no pkg.json found')
package = Package.fromfile(package, directory) self.data = {'name': filename[:-5]}
try: if self.model == Package:
result = subprocess.Popen(['git', '-C', path, 'rev-parse', '--verify', 'HEAD'], stdout=subprocess.PIPE) self.data['commit_id'] = None
returncode = result.wait() try:
except: full_package_dir = os.path.join(settings.MAP_ROOT, package_dir)
pass result = subprocess.Popen(['git', '-C', full_package_dir, 'rev-parse', '--verify', 'HEAD'],
else: stdout=subprocess.PIPE)
if returncode == 0: returncode = result.wait()
package['commit_id'] = result.stdout.read().strip() except FileNotFoundError:
pass
else:
if returncode == 0:
self.data['commit_id'] = result.stdout.read().strip()
objects.add_package(package) try:
objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels'))) add_data = self.model.fromfile(self.json_data)
objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True)) except Exception as e:
return objects raise CommandError('Could not load data: %s' % e)
self.data.update(add_data)
relations = {
'level': Level,
}
def _read_folder(package, cls, path, check_sister_file=False): def save(self):
objects = [] if self.model != Package:
if not os.path.isdir(path): package_name = self.reader.package_names_by_dir[self.package_dir]
return [] self.data['package'] = self.reader.saved_items[Package][package_name].obj
for filename in sorted(os.listdir(path)):
if not filename.endswith('.json'):
continue
full_filename = os.path.join(path, filename) # Change name references to the referenced object
if not os.path.isfile(full_filename): for name, model in self.relations.items():
continue if name in self.data:
self.data[name] = self.reader.saved_items[model][self.data[name]].obj
name = filename[:-5] obj, created = self.model.objects.update_or_create(name=self.data['name'], defaults=self.data)
if check_sister_file and os.path.isfile(name): if created:
raise CommandError('%s: %s is missing.' % (filename, name)) print('- Created %s: %s' % (self.model.__name__, obj.name))
objects.append(cls.fromfile(json.load(open(full_filename)), package, name)) self.obj = obj
return objects self.reader.saved_items[self.model][obj.name] = self

View file

@ -1,98 +0,0 @@
import json
from django.core.management.base import CommandError
from ..models import Level, Package, Source
from ..utils import json_encoder_reindent
class ObjectCollection:
def __init__(self):
self.packages = {}
self.levels = {}
self.sources = {}
def add_package(self, package):
self._add(self.packages, 'package', package)
def add_level(self, level):
self._add(self.levels, 'level', level)
def add_source(self, source):
self._add(self.sources, 'source', source)
def add_packages(self, packages):
for package in packages:
self.add_package(package)
def add_levels(self, levels):
for level in levels:
self.add_level(level)
def add_sources(self, sources):
for source in sources:
self.add_source(source)
def _add(self, container, name, item):
if item['name'] in container:
raise CommandError('Duplicate %s name: %s' % (name, item['name']))
container[item['name']] = item
def apply_to_db(self):
for name, package in tuple(self.packages.items()):
for depname in package['depends']:
if depname not in self.packages:
raise CommandError('Missing dependency: %s' % depname)
for name, package in tuple(self.packages.items()):
package = package.copy()
orig_deps = package.pop('depends', [])
package, created = Package.objects.update_or_create(name=name, defaults=package)
package.orig_deps = orig_deps
self.packages[name] = package
if created:
print('- Created package: '+name)
for name, level in self.levels.items():
level['package'] = self.packages[level['package']]
level, created = Level.objects.update_or_create(name=name, defaults=level)
self.levels[name] = level
if created:
print('- Created level: '+name)
for name, source in self.sources.items():
source['package'] = self.packages[source['package']]
source, created = Source.objects.update_or_create(name=name, defaults=source)
self.sources[name] = source
if created:
print('- Created source: '+name)
for source in Source.objects.exclude(name__in=self.sources.keys()):
print('- Deleted source: '+source.name)
source.delete()
for level in Level.objects.exclude(name__in=self.levels.keys()):
print('- Deleted level: '+level.name)
level.delete()
for package in Package.objects.exclude(name__in=self.packages.keys()):
print('- Deleted package: '+package.name)
package.delete()
for name, package in tuple(self.packages.items()):
has_deps = []
for dependency in tuple(package.depends.all()):
if dependency.name not in package.orig_deps:
package.depends.remove(dependency)
print('- Removed dependency: '+dependency.name)
else:
has_deps.append(dependency.name)
for depname in package.orig_deps:
if depname not in has_deps:
package.depends.add(self.packages[depname])
print('- Added dependency: '+depname)
def json_encode(data):
return json_encoder_reindent(json.dumps, data, indent=4)+'\n'

View file

@ -7,8 +7,9 @@ from datetime import datetime
from django.conf import settings from django.conf import settings
from django.utils import timezone from django.utils import timezone
from c3nav.mapdata.utils import json_encoder_reindent
from ..models import Package from ..models import Package
from .utils import json_encode
def write_packages(prettify=False, check_only=False): def write_packages(prettify=False, check_only=False):
@ -109,3 +110,7 @@ def _write_object(obj, path, filename, prettify=False, check_only=False):
with open(full_filename, 'w') as f: with open(full_filename, 'w') as f:
f.write(new_data_encoded) f.write(new_data_encoded)
return 1 return 1
def json_encode(data):
return json_encoder_reindent(json.dumps, data, indent=4)+'\n'