2025-summer/team-3
12
Fork 0
Code Issues Pull requests Projects Releases Activity

refactor models, change map package storage and map package loading

Browse source
This commit is contained in:
Laura Klünder 2016-08-29 22:10:43 +02:00
parent 0656090f27
commit 376d85f9b4
12 changed files with 226 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/c3nav/control/admin.py
View file

@ -1,3 +1,3 @@
from django.contrib import admin
from django.contrib import admin # noqa
# Register your models here.

2
src/c3nav/control/models.py
View file

@ -1,3 +1,3 @@
from django.db import models
from django.db import models # noqa
# Create your models here.

4
src/c3nav/control/templates/control/editor.html
View file

@ -19,9 +19,9 @@ var map = L.map('mapeditor', {
});
// Add Source Layers
{% for pkg in packages %}
{% for source_list in sources %}
L.control.layers([], {
{% for source in pkg.sources.all %}
{% for source in source_list %}
"{{ source.name }}": L.imageOverlay('{% url 'map.source' source=source.name %}', {{ source.jsbounds }}),{% endfor %}
}).addTo(map);
{% endfor %}

2
src/c3nav/control/views.py
View file

@ -19,7 +19,7 @@ def editor(request, level=None):
level = get_object_or_404(Level, name=level)
return render(request, 'control/editor.html', {
'bounds': json.dumps(Source.max_bounds()),
'packages': Package.objects.all(),
'sources': [p.sources.all().order_by('name') for p in Package.objects.all()],
'levels': Level.objects.all(),
'current_level': level,
})

8
src/c3nav/mapdata/management/commands/loadmappkgs.py → src/c3nav/mapdata/management/commands/loadmap.py
View file

@ -1,19 +1,19 @@
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from ...packageio import MapPackagesIO
from ...packageio import read_packages
class Command(BaseCommand):
help = 'Load the given map packages into the database'
help = 'Update the map database'
def add_arguments(self, parser):
parser.add_argument('mappkgdir', nargs='+', type=str, help='map package directories')
parser.add_argument('-y', action='store_const', const=True, default=False,
help='don\'t ask for confirmation')
def handle(self, *args, **options):
with transaction.atomic():
MapPackagesIO(options['mappkgdir']).update_to_db()
read_packages()
print()
if input('Confirm (y/N): ') != 'y':
raise CommandError('Aborted.')

9
src/c3nav/mapdata/migrations/0001_initial.py
View file

@ -1,11 +1,10 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.9 on 2016-08-29 16:48
# Generated by Django 1.9.9 on 2016-08-29 20:00
from __future__ import unicode_literals
import django.db.models.deletion
from django.db import migrations, models
import c3nav.mapdata.models.source
import parler.models
@ -39,10 +38,10 @@ class Migration(migrations.Migration):
('master', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='mapdata.Feature')),
],
options={
'db_tablespace': '',
'managed': True,
'default_permissions': (),
'db_table': 'mapdata_feature_translation',
'managed': True,
'db_tablespace': '',
'verbose_name': 'feature Translation',
},
),
@ -66,6 +65,7 @@ class Migration(migrations.Migration):
('left', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='left coordinate')),
('top', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='top coordinate')),
('right', models.DecimalField(decimal_places=2, max_digits=6, null=True, verbose_name='right coordinate')),
('directory', models.CharField(max_length=100, verbose_name='folder name')),
],
),
migrations.CreateModel(
@ -73,7 +73,6 @@ class Migration(migrations.Migration):
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.SlugField(unique=True, verbose_name='source name')),
('image', models.FileField(max_length=70, storage=c3nav.mapdata.models.source.SourceImageStorage(), upload_to=c3nav.mapdata.models.source.map_source_filename, verbose_name='source image')),
('bottom', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='bottom coordinate')),
('left', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='left coordinate')),
('top', models.DecimalField(decimal_places=2, max_digits=6, verbose_name='top coordinate')),

22
src/c3nav/mapdata/models/level.py
View file

@ -1,3 +1,5 @@
from collections import OrderedDict
from django.db import models
from django.utils.translation import ugettext_lazy as _
@ -12,5 +14,25 @@ class Level(models.Model):
package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='levels',
verbose_name=_('map package'))
def jsonize(self):
return OrderedDict((
('name', self.name),
('altitude', float(self.altitude)),
))
@classmethod
def fromfile(cls, data, package, name):
if 'altitude' not in data:
raise ValueError('%s.json: missing altitude.' % name)
if not isinstance(data['altitude'], (int, float)):
raise ValueError('%s.json: altitude has to be in or float.')
return {
'package': package,
'name': name,
'altitude': data['altitude'],
}
class Meta:
ordering = ['altitude']

34
src/c3nav/mapdata/models/package.py
View file

@ -1,3 +1,5 @@
from collections import OrderedDict
from django.db import models
from django.utils.translation import ugettext_lazy as _
@ -13,3 +15,35 @@ class Package(models.Model):
left = models.DecimalField(_('left coordinate'), null=True, max_digits=6, decimal_places=2)
top = models.DecimalField(_('top coordinate'), null=True, max_digits=6, decimal_places=2)
right = models.DecimalField(_('right coordinate'), null=True, max_digits=6, decimal_places=2)
directory = models.CharField(_('folder name'), max_length=100)
@classmethod
def fromfile(cls, data, directory):
kwargs = {
'directory': directory
}
if 'name' not in data:
raise ValueError('pkg.json: missing package name.')
kwargs['name'] = data['name']
if 'bounds' in data:
bounds = data['bounds']
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise ValueError('pkg.json: Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise ValueError('pkg.json: All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise ValueError('pkg.json: bounds: lower coordinate has to be first.')
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
return kwargs
def tofile(self):
data = OrderedDict()
data['name'] = self.name
if self.bottom is not None:
data['bounds'] = ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))
return data

60
src/c3nav/mapdata/models/source.py
View file

@ -1,24 +1,10 @@
import json
import os
from collections import OrderedDict
from django.conf import settings
from django.core.files.storage import FileSystemStorage
from django.db import models, transaction
from django.dispatch import receiver
from django.db import models
from django.utils.translation import ugettext_lazy as _
class SourceImageStorage(FileSystemStorage):
def get_available_name(self, name, *args, max_length=None, **kwargs):
if self.exists(name):
os.remove(os.path.join(settings.MEDIA_ROOT, name))
return super().get_available_name(name, *args, max_length, **kwargs)
def map_source_filename(instance, filename):
return os.path.join('mapsources', '%s.%s' % (instance.name, filename.split('.')[-1]))
class Source(models.Model):
"""
A map source, images of levels that can be useful as backgrounds for the map editor
@ -27,9 +13,6 @@ class Source(models.Model):
package = models.ForeignKey('Package', on_delete=models.CASCADE, related_name='sources',
verbose_name=_('map package'))
image = models.FileField(_('source image'), max_length=70,
upload_to=map_source_filename, storage=SourceImageStorage())
bottom = models.DecimalField(_('bottom coordinate'), max_digits=6, decimal_places=2)
left = models.DecimalField(_('left coordinate'), max_digits=6, decimal_places=2)
top = models.DecimalField(_('top coordinate'), max_digits=6, decimal_places=2)
@ -50,23 +33,30 @@ class Source(models.Model):
def jsbounds(self):
return json.dumps(((float(self.bottom), float(self.left)), (float(self.top), float(self.right))))
@classmethod
def fromfile(cls, data, package, name):
kwargs = {
'package': package,
'name': name,
}
@receiver(models.signals.post_delete, sender=Source)
def delete_image_on_mapsource_delete(sender, instance, **kwargs):
transaction.on_commit(lambda: instance.image.delete(save=False))
if 'bounds' not in data:
raise ValueError('%s.json: missing bounds.' % name)
bounds = data['bounds']
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise ValueError('pkg.json: Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise ValueError('pkg.json: All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise ValueError('pkg.json: bounds: lower coordinate has to be first.')
(kwargs['bottom'], kwargs['left']), (kwargs['top'], kwargs['right']) = bounds
@receiver(models.signals.pre_save, sender=Source)
def delete_image_on_mapsource_change(sender, instance, **kwargs):
if not instance.pk:
return False
return kwargs
try:
old_file = Source.objects.get(pk=instance.pk).image
except Source.DoesNotExist:
return False
new_file = instance.image
if map_source_filename(instance, new_file.name) != old_file.name:
transaction.on_commit(lambda: old_file.delete(save=False))
def jsonize(self):
return OrderedDict((
('name', self.name),
('src', 'sources/'+self.get_export_filename()),
('bounds', ((float(self.bottom), float(self.left)), (float(self.top), float(self.right)))),
))

220
src/c3nav/mapdata/packageio.py
View file

@ -1,133 +1,157 @@
import json
import os
from collections import OrderedDict
from django.core.files import File
from django.conf import settings
from django.core.management.base import CommandError
from .models import Level, Package, Source
class PackageIOError(CommandError):
pass
class ObjectCollection:
def __init__(self):
self.packages = {}
self.levels = {}
self.sources = {}
def add_package(self, package):
self._add(self.packages, 'package', package)
class MapPackagesIO():
def __init__(self, directories):
print('Opening Map Packages…')
self.packages = OrderedDict()
self.levels = OrderedDict()
self.sources = OrderedDict()
def add_level(self, level):
self._add(self.levels, 'level', level)
for directory in directories:
print('- '+directory)
def add_source(self, source):
self._add(self.sources, 'source', source)
try:
package = json.load(open(os.path.join(directory, 'pkg.json')))
except FileNotFoundError:
raise PackageIOError('no pkg.json found in %s' % directory)
def add_packages(self, packages):
for package in packages:
self.add_package(package)
if package['name'] in self.packages:
raise PackageIOError('Duplicate package name: %s' % package['name'])
def add_levels(self, levels):
for level in levels:
self.add_level(level)
if 'bounds' in package:
self._validate_bounds(package['bounds'])
def add_sources(self, sources):
for source in sources:
self.add_source(source)
package['directory'] = directory
self.packages[package['name']] = package
def _add(self, container, name, item):
if item['name'] in container:
raise CommandError('Duplicate %s name: %s' % (name, item['name']))
container[item['name']] = item
for level in package.get('levels', []):
level = level.copy()
if level['name'] in self.levels:
raise PackageIOError('Duplicate level name: %s in packages %s and %s' %
(level, self.levels[level]['name'], package['name']))
if not isinstance(level['altitude'], (int, float)):
raise PackageIOError('levels: %s: altitude has to be int or float.' % level['name'])
level['package'] = package['name']
self.levels[level['name']] = level
for source in package.get('sources', []):
source = source.copy()
if source['name'] in self.sources:
raise PackageIOError('Duplicate source name: %s in packages %s and %s' %
(source['name'], self.sources[source['name']]['name'], package['name']))
self._validate_bounds(source['bounds'], 'sources: %s: ' % source['name'])
source['filename'] = os.path.join(directory, source['src'])
if not os.path.isfile(source['filename']):
raise PackageIOError('Source file not found: '+source['filename'])
source['package'] = package['name']
self.sources[source['name']] = source
def _validate_bounds(self, bounds, prefix=''):
if len(bounds) != 2 or len(bounds[0]) != 2 or len(bounds[1]) != 2:
raise PackageIOError(prefix+'Invalid bounds format.')
if not all(isinstance(i, (float, int)) for i in sum(bounds, [])):
raise PackageIOError(prefix+'All bounds coordinates have to be int or float.')
if bounds[0][0] >= bounds[1][0] or bounds[0][1] >= bounds[1][1]:
raise PackageIOError(prefix+'bounds: lower coordinate has to be first.')
def update_to_db(self):
from .models import MapPackage, MapLevel, MapSource
print('Updating Map database…')
# Add new Packages
packages = {}
print('- Updating packages…')
for name, package in self.packages.items():
bounds = package.get('bounds')
defaults = {
'bottom': bounds[0][0],
'left': bounds[0][1],
'top': bounds[1][0],
'right': bounds[1][1],
} if bounds else {}
package, created = MapPackage.objects.update_or_create(name=name, defaults=defaults)
packages[name] = package
def apply_to_db(self):
for name, package in tuple(self.packages.items()):
package, created = Package.objects.update_or_create(name=name, defaults=package)
self.packages[name] = package
if created:
print('- Created package: '+name)
# Add new levels
print('- Updating levels…')
for name, level in self.levels.items():
package, created = MapLevel.objects.update_or_create(name=name, defaults={
'package': packages[level['package']],
'altitude': level['altitude'],
'name': level['name'],
})
level['package'] = self.packages[level['package']]
level, created = Level.objects.update_or_create(name=name, defaults=level)
self.levels[name] = level
if created:
print('- Created level: '+name)
# Add new map sources
print('- Updating sources…')
for name, source in self.sources.items():
source, created = MapSource.objects.update_or_create(name=name, defaults={
'package': packages[source['package']],
'image': File(open(source['filename'], 'rb')),
'bottom': source['bounds'][0][0],
'left': source['bounds'][0][1],
'top': source['bounds'][1][0],
'right': source['bounds'][1][1],
})
source['package'] = self.packages[source['package']]
source, created = Source.objects.update_or_create(name=name, defaults=source)
self.sources[name] = source
if created:
print('- Created source: '+name)
# Remove old sources
for source in MapSource.objects.exclude(name__in=self.sources.keys()):
for source in Source.objects.exclude(name__in=self.sources.keys()):
print('- Deleted source: '+source.name)
source.delete()
# Remove old levels
for level in MapLevel.objects.exclude(name__in=self.levels.keys()):
for level in Level.objects.exclude(name__in=self.levels.keys()):
print('- Deleted level: '+level.name)
level.delete()
# Remove old packages
for package in MapPackage.objects.exclude(name__in=self.packages.keys()):
for package in Package.objects.exclude(name__in=self.packages.keys()):
print('- Deleted package: '+package.name)
package.delete()
def read_packages():
print('Detecting Map Packages…')
objects = ObjectCollection()
for directory in os.listdir(settings.MAP_ROOT):
print('\n'+directory)
if not os.path.isdir(os.path.join(settings.MAP_ROOT, directory)):
continue
read_package(directory, objects)
objects.apply_to_db()
def read_package(directory, objects=None):
if objects is None:
objects = ObjectCollection()
path = os.path.join(settings.MAP_ROOT, directory)
# Main JSON
try:
package = json.load(open(os.path.join(path, 'pkg.json')))
except FileNotFoundError:
raise CommandError('no pkg.json found')
package = Package.fromfile(package, directory)
objects.add_package(package)
objects.add_levels(_read_folder(package['name'], Level, os.path.join(path, 'levels')))
objects.add_sources(_read_folder(package['name'], Source, os.path.join(path, 'sources'), check_sister_file=True))
return objects
def _read_folder(package, cls, path, check_sister_file=False):
objects = []
if not os.path.isdir(path):
return []
for filename in os.listdir(path):
if not filename.endswith('.json'):
continue
full_filename = os.path.join(path, filename)
if not os.path.isfile(full_filename):
continue
name = filename[:-5]
if check_sister_file and os.path.isfile(name):
raise CommandError('%s: %s is missing.' % (filename, name))
objects.append(cls.fromfile(json.load(open(full_filename)), package, name))
return objects
def _fromfile_validate(cls, data, name):
obj = cls.fromfile(json.loads(data), name=name)
formatted_data = json_encode(obj.tofile())
if data != formatted_data:
raise CommandError('%s.json is not correctly formatted, its contents are:\n---\n' +
data+'\n---\nbut they should be\n---\n'+formatted_data+'\n---')
def _json_encode_preencode(data, magic_marker):
if isinstance(data, dict):
data = data.copy()
for name, value in tuple(data.items()):
if name in ('bounds', ):
data[name] = magic_marker+json.dumps(value)+magic_marker
else:
data[name] = _json_encode_preencode(value, magic_marker)
return data
elif isinstance(data, (tuple, list)):
return tuple(_json_encode_preencode(value, magic_marker) for value in data)
else:
return data
def json_encode(data):
magic_marker = '***JSON_MAGIC_MARKER***'
test_encode = json.dumps(data)
while magic_marker in test_encode:
magic_marker += '*'
result = json.dumps(_json_encode_preencode(data, magic_marker), indent=4)
return result.replace('"'+magic_marker, '').replace(magic_marker+'"', '')+'\n'

10
src/c3nav/mapdata/views.py
View file

@ -1,8 +1,11 @@
import mimetypes
import os
from django.conf import settings
from django.contrib.admin.views.decorators import staff_member_required
from django.core.files import File
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render
from django.shortcuts import get_object_or_404
from .models import Source
@ -10,7 +13,8 @@ from .models import Source
@staff_member_required
def source(request, source):
source = get_object_or_404(Source, name=source)
response = HttpResponse(content_type=mimetypes.guess_type(source.image.name)[0])
for chunk in source.image.chunks():
response = HttpResponse(content_type=mimetypes.guess_type(source.name)[0])
image_path = os.path.join(settings.MAP_ROOT, source.package.directory, 'sources', source.name)
for chunk in File(open(image_path, 'rb')).chunks():
response.write(chunk)
return response

3
src/c3nav/settings.py
View file

@ -17,6 +17,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = config.get('c3nav', 'datadir', fallback=os.environ.get('DATA_DIR', 'data'))
LOG_DIR = os.path.join(DATA_DIR, 'logs')
MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
MAP_ROOT = os.path.join(DATA_DIR, 'map')
if not os.path.exists(DATA_DIR):
os.mkdir(DATA_DIR)
@ -24,6 +25,8 @@ if not os.path.exists(LOG_DIR):
os.mkdir(LOG_DIR)
if not os.path.exists(MEDIA_ROOT):
os.mkdir(MEDIA_ROOT)
if not os.path.exists(MAP_ROOT):
os.mkdir(MAP_ROOT)
if config.has_option('django', 'secret'):
SECRET_KEY = config.get('django', 'secret')