rewrite loadmap implementation

This commit is contained in:
Laura Klünder 2016-09-24 14:09:52 +02:00
parent e074f70799
commit 995ccffc67
9 changed files with 190 additions and 182 deletions

View file

@ -1,7 +1,7 @@
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from ...packageio import read_packages
from ...packageio import MapdataReader
class Command(BaseCommand):
@ -12,8 +12,11 @@ class Command(BaseCommand):
help='don\'t ask for confirmation')
def handle(self, *args, **options):
reader = MapdataReader()
reader.read_packages()
with transaction.atomic():
read_packages()
reader.apply_to_db()
print()
if not options['yes'] and input('Confirm (y/N): ') != 'y':
raise CommandError('Aborted.')

View file

@ -11,6 +11,7 @@ from ..fields import GeometryField
class FeatureType(namedtuple('FeatureType', ('name', 'title', 'title_plural', 'geomtype', 'color'))):
# noinspection PyUnusedLocal
def __init__(self, *args, **kwargs):
super().__init__()
FEATURE_TYPES[self.name] = self
@ -42,6 +43,8 @@ class Feature(models.Model):
verbose_name=_('level'))
geometry = GeometryField()
path_regex = r'^features/('+'|'.join(name for name, title in TYPES)+')/'
@property
def titles(self):
return {title.language: title.title for title in self.featuretitles.all()}

View file

@ -1,6 +1,8 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.models import Package
class Level(models.Model):
"""
@ -9,20 +11,20 @@ class Level(models.Model):
name = models.SlugField(_('level name'), primary_key=True, max_length=50,
help_text=_('Usually just an integer (e.g. -1, 0, 1, 2)'))
altitude = models.DecimalField(_('level altitude'), null=True, max_digits=6, decimal_places=2)
package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels',
package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='levels',
verbose_name=_('map package'))
path_regex = r'^levels/'
@classmethod
def fromfile(cls, data, package, name):
def fromfile(cls, data):
if 'altitude' not in data:
raise ValueError('%s.json: missing altitude.' % name)
raise ValueError('missing altitude.')
if not isinstance(data['altitude'], (int, float)):
raise ValueError('%s.json: altitude has to be in or float.')
raise ValueError('altitude has to be int or float.')
return {
'name': name,
'package': package,
'altitude': data['altitude'],
}

View file

@ -22,32 +22,33 @@ class Package(models.Model):
directory = models.CharField(_('folder name'), max_length=100)
path_regex = r'^package.json$'
@classmethod
def fromfile(cls, data, directory):
kwargs = {
'directory': directory
}
def fromfile(cls, data):
kwargs = {}
if 'name' not in data:
raise ValueError('pkg.json: missing package name.')
raise ValueError('missing package name.')
kwargs['name'] = data['name']
depends = data.get('depends', [])
if not isinstance(depends, list):
raise TypeError('pkg.json: depends has to be a list')
raise TypeError('depends has to be a list')
kwargs['depends'] = depends
if 'home_repo' in data:
kwargs['home_repo'] = data['home_repo']
kwargs['home_repo'] = data['home_repo'] if 'home_repo' in data else None
if 'bounds' in data:
bounds = data['bounds']
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise ValueError('pkg.json: Invalid bounds format.')
raise ValueError('Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise ValueError('pkg.json: All bounds coordinates have to be int or float.')
raise ValueError('All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise ValueError('pkg.json: bounds: lower coordinate has to be first.')
raise ValueError('bounds: lower coordinate has to be first.')
else:
bounds = (None, None), (None, None)
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
return kwargs
@ -64,7 +65,7 @@ class Package(models.Model):
def bounds(self):
if self.bottom is None:
return None
return ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))
return (float(self.bottom), float(self.left)), (float(self.top), float(self.right))
def tofile(self):
data = OrderedDict()

View file

@ -1,13 +1,15 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.models import Package
class Source(models.Model):
"""
A map source, images of levels that can be useful as backgrounds for the map editor
"""
name = models.SlugField(_('source name'), primary_key=True, max_length=50)
package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='sources',
package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name='sources',
verbose_name=_('map package'))
bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2)
@ -15,6 +17,8 @@ class Source(models.Model):
top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2)
right = models.DecimalField(_('right coordinate'), max_digits=6, decimal_places=2)
path_regex = r'^sources/'
@classmethod
def max_bounds(cls):
result = cls.objects.all().aggregate(models.Min('bottom'), models.Min('left'),
@ -24,25 +28,22 @@ class Source(models.Model):
@property
def bounds(self):
return ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))
return (float(self.bottom), float(self.left)), (float(self.top), float(self.right))
@classmethod
def fromfile(cls, data, package, name):
kwargs = {
'package': package,
'name': name,
}
def fromfile(cls, data):
kwargs = {}
if 'bounds' not in data:
raise ValueError('%s.json: missing bounds.' % name)
raise ValueError('missing bounds.')
bounds = data['bounds']
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise ValueError('pkg.json: Invalid bounds format.')
raise ValueError('Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise ValueError('pkg.json: All bounds coordinates have to be int or float.')
raise ValueError('All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise ValueError('pkg.json: bounds: lower coordinate has to be first.')
raise ValueError('bounds: lower coordinate has to be first.')
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
return kwargs

View file

@ -1,3 +1,2 @@
from .read import read_packages, read_package # noqa
from .read import MapdataReader # noqa
from .write import write_packages, write_package # noqa
from .utils import ObjectCollection # noqa

View file

@ -1,71 +1,163 @@
import json
import os
import re
import subprocess
from django.conf import settings
from django.core.management.base import CommandError
from django.core.management import CommandError
from ..models import Level, Package, Source
from .utils import ObjectCollection
from ..models import Feature, Level, Package, Source
def read_packages():
class MapdataReader:
ordered_models = (Package, Level, Source, Feature)
def __init__(self):
self.content = {}
self.package_names_by_dir = {}
self.saved_items = {model: {} for model in self.ordered_models}
def read_packages(self):
print('Detecting Map Packages…')
objects = ObjectCollection()
for directory in os.listdir(settings.MAP_ROOT):
print('\n'+directory)
print('\n' + directory)
if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)):
continue
read_package(directory, objects)
self.read_package(directory)
objects.apply_to_db()
def read_package(self, package_dir):
full_package_dir = os.path.join(settings.MAP_ROOT, package_dir)
for path, sub_dirs, filenames in os.walk(full_package_dir):
sub_dirs[:] = sorted([directory for directory in sub_dirs if not directory.startswith('.')])
for filename in sorted(filenames):
if not filename.endswith('.json'):
continue
self.add_file(package_dir, path[len(full_package_dir) + 1:], filename)
def _add_item(self, item):
if item.package_dir not in self.content:
self.content[item.package_dir] = {model: [] for model in self.ordered_models}
self.content[item.package_dir][item.model].append(item)
def add_file(self, package_dir, path, filename):
file_path = os.path.join(package_dir, path, filename)
relative_file_path = os.path.join(path, filename)
print(file_path)
for model in self.ordered_models:
if re.search(model.path_regex, relative_file_path):
self._add_item(ReaderItem(self, package_dir, path, filename, model))
break
else:
raise CommandError('Unexpected JSON file: %s' % file_path)
def apply_to_db(self):
# Collect all Packages
package_items_by_name = {}
package_dirs_by_name = {}
for package_dir, items_by_model in self.content.items():
if not items_by_model[Package]:
raise CommandError('Missing package file: %s' % package_dir)
if len(items_by_model[Package]) > 1:
raise CommandError('Multiple package files: %s' % package_dir)
package_item = items_by_model[Package][0]
package_items_by_name[package_item.data['name']] = package_item
package_dirs_by_name[package_item.data['name']] = package_dir
self.package_names_by_dir[package_dir] = package_item.data['name']
# Resolve Package Dependencies
unresolved_packages = set(package_items_by_name.keys())
resolved_packages = set()
package_order = []
while unresolved_packages:
resolvable = set([name for name in unresolved_packages if
not set(package_items_by_name[name].data['depends'])-resolved_packages])
if not resolvable:
raise CommandError('Could not resolve package dependencies: %s' % unresolved_packages)
package_order.extend(resolvable)
unresolved_packages -= resolvable
resolved_packages |= resolvable
# Create new and update existing entries
for package_name in package_order:
print('')
package_dir = package_dirs_by_name[package_name]
items_by_model = self.content[package_dir]
for model in self.ordered_models:
items = items_by_model[model]
for item in items:
item.save()
# Delete old entries
for model in reversed(self.ordered_models):
queryset = model.objects.exclude(name__in=self.saved_items[model].keys())
for name in queryset.values_list('name', flat=True):
print('- Deleted %s: %s' % (model.__name__, name))
queryset.delete()
def read_package(directory, objects=None):
if objects is None:
objects = ObjectCollection()
path = os.path.join(settings.MAP_ROOT, directory)
# Main JSON
try:
package = json.load(open(os.path.join(path, 'pkg.json')))
except FileNotFoundError:
raise CommandError('no pkg.json found')
package = Package.fromfile(package, directory)
class ReaderItem:
def __init__(self, reader, package_dir, path, filename, model):
self.reader = reader
self.package_dir = package_dir
self.path = path
self.filename = filename
self.model = model
self.obj = None
self.path_in_package = os.path.join(self.path, self.filename)
try:
result = subprocess.Popen(['git', '-C', path, 'rev-parse', '--verify', 'HEAD'], stdout=subprocess.PIPE)
with open(os.path.join(settings.MAP_ROOT, package_dir, path, filename)) as f:
self.content = f.read()
except Exception as e:
raise CommandError('Could not read File: %s' % e)
try:
self.json_data = json.loads(self.content)
except json.JSONDecodeError as e:
raise CommandError('Could not decode JSON: %s' % e)
self.data = {'name': filename[:-5]}
if self.model == Package:
self.data['commit_id'] = None
try:
full_package_dir = os.path.join(settings.MAP_ROOT, package_dir)
result = subprocess.Popen(['git', '-C', full_package_dir, 'rev-parse', '--verify', 'HEAD'],
stdout=subprocess.PIPE)
returncode = result.wait()
except:
except FileNotFoundError:
pass
else:
if returncode == 0:
package['commit_id'] = result.stdout.read().strip()
self.data['commit_id'] = result.stdout.read().strip()
objects.add_package(package)
objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels')))
objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True))
return objects
try:
add_data = self.model.fromfile(self.json_data)
except Exception as e:
raise CommandError('Could not load data: %s' % e)
self.data.update(add_data)
relations = {
'level': Level,
}
def _read_folder(package, cls, path, check_sister_file=False):
objects = []
if not os.path.isdir(path):
return []
for filename in sorted(os.listdir(path)):
if not filename.endswith('.json'):
continue
def save(self):
if self.model != Package:
package_name = self.reader.package_names_by_dir[self.package_dir]
self.data['package'] = self.reader.saved_items[Package][package_name].obj
full_filename = os.path.join(path, filename)
if not os.path.isfile(full_filename):
continue
# Change name references to the referenced object
for name, model in self.relations.items():
if name in self.data:
self.data[name] = self.reader.saved_items[model][self.data[name]].obj
name = filename[:-5]
if check_sister_file and os.path.isfile(name):
raise CommandError('%s: %s is missing.' % (filename, name))
obj, created = self.model.objects.update_or_create(name=self.data['name'], defaults=self.data)
if created:
print('- Created %s: %s' % (self.model.__name__, obj.name))
objects.append(cls.fromfile(json.load(open(full_filename)), package, name))
return objects
self.obj = obj
self.reader.saved_items[self.model][obj.name] = self

View file

@ -1,98 +0,0 @@
import json
from django.core.management.base import CommandError
from ..models import Level, Package, Source
from ..utils import json_encoder_reindent
class ObjectCollection:
def __init__(self):
self.packages = {}
self.levels = {}
self.sources = {}
def add_package(self, package):
self._add(self.packages, 'package', package)
def add_level(self, level):
self._add(self.levels, 'level', level)
def add_source(self, source):
self._add(self.sources, 'source', source)
def add_packages(self, packages):
for package in packages:
self.add_package(package)
def add_levels(self, levels):
for level in levels:
self.add_level(level)
def add_sources(self, sources):
for source in sources:
self.add_source(source)
def _add(self, container, name, item):
if item['name'] in container:
raise CommandError('Duplicate %s name: %s' % (name, item['name']))
container[item['name']] = item
def apply_to_db(self):
for name, package in tuple(self.packages.items()):
for depname in package['depends']:
if depname not in self.packages:
raise CommandError('Missing dependency: %s' % depname)
for name, package in tuple(self.packages.items()):
package = package.copy()
orig_deps = package.pop('depends', [])
package, created = Package.objects.update_or_create(name=name, defaults=package)
package.orig_deps = orig_deps
self.packages[name] = package
if created:
print('- Created package: '+name)
for name, level in self.levels.items():
level['package'] = self.packages[level['package']]
level, created = Level.objects.update_or_create(name=name, defaults=level)
self.levels[name] = level
if created:
print('- Created level: '+name)
for name, source in self.sources.items():
source['package'] = self.packages[source['package']]
source, created = Source.objects.update_or_create(name=name, defaults=source)
self.sources[name] = source
if created:
print('- Created source: '+name)
for source in Source.objects.exclude(name__in=self.sources.keys()):
print('- Deleted source: '+source.name)
source.delete()
for level in Level.objects.exclude(name__in=self.levels.keys()):
print('- Deleted level: '+level.name)
level.delete()
for package in Package.objects.exclude(name__in=self.packages.keys()):
print('- Deleted package: '+package.name)
package.delete()
for name, package in tuple(self.packages.items()):
has_deps = []
for dependency in tuple(package.depends.all()):
if dependency.name not in package.orig_deps:
package.depends.remove(dependency)
print('- Removed dependency: '+dependency.name)
else:
has_deps.append(dependency.name)
for depname in package.orig_deps:
if depname not in has_deps:
package.depends.add(self.packages[depname])
print('- Added dependency: '+depname)
def json_encode(data):
return json_encoder_reindent(json.dumps, data, indent=4)+'\n'

View file

@ -7,8 +7,9 @@ from datetime import datetime
from django.conf import settings
from django.utils import timezone
from c3nav.mapdata.utils import json_encoder_reindent
from ..models import Package
from .utils import json_encode
def write_packages(prettify=False, check_only=False):
@ -109,3 +110,7 @@ def _write_object(obj, path, filename, prettify=False, check_only=False):
with open(full_filename, 'w') as f:
f.write(new_data_encoded)
return 1
def json_encode(data):
return json_encoder_reindent(json.dumps, data, indent=4)+'\n'