From 24a2fe95227bf918bd49355f580d4c5700144e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laura=20Kl=C3=BCnder?= Date: Tue, 24 Oct 2017 00:11:02 +0200 Subject: [PATCH] write binary map history index --- src/c3nav/editor/models/changeset.py | 4 +- src/c3nav/mapdata/cache.py | 49 ++++++++++++++++++----- src/c3nav/mapdata/models/geometry/base.py | 13 +++++- src/c3nav/mapdata/models/update.py | 12 ++++-- src/c3nav/settings.py | 3 ++ 5 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/c3nav/editor/models/changeset.py b/src/c3nav/editor/models/changeset.py index 5f9a6670..93ca904f 100644 --- a/src/c3nav/editor/models/changeset.py +++ b/src/c3nav/editor/models/changeset.py @@ -20,6 +20,7 @@ from django.utils.translation import ungettext_lazy from c3nav.editor.models.changedobject import ApplyToInstanceError, ChangedObject from c3nav.editor.utils import is_created_pk from c3nav.editor.wrappers import ModelInstanceWrapper, ModelWrapper +from c3nav.mapdata.cache import changed_geometries from c3nav.mapdata.models import LocationSlug, MapUpdate from c3nav.mapdata.models.locations import LocationRedirect from c3nav.mapdata.utils.models import get_submodels @@ -453,11 +454,12 @@ class ChangeSet(models.Model): changeset.save() elif self.direct_editing: with MapUpdate.lock(): + changed_geometries.reset() queries_before = len(connection.queries) yield self if any((q['sql'].startswith('UPDATE') or q['sql'].startswith('INSERT') or q['sql'].startswith('DELETE')) for q in connection.queries[queries_before:]): - MapUpdate.objects.create(user=user, type='direct_edit') + update = MapUpdate.objects.create(user=user, type='direct_edit') else: yield self diff --git a/src/c3nav/mapdata/cache.py b/src/c3nav/mapdata/cache.py index 5f70696b..d4d64303 100644 --- a/src/c3nav/mapdata/cache.py +++ b/src/c3nav/mapdata/cache.py @@ -1,4 +1,5 @@ import math +import os import struct import numpy as np @@ -6,6 +7,7 @@ from django.conf import settings from django.db.models.signals import m2m_changed, post_delete from shapely import prepared from shapely.geometry import box +from shapely.ops import unary_union from c3nav.mapdata.utils.models import get_submodels @@ -23,8 +25,8 @@ class MapHistory: def __init__(self, resolution=settings.CACHE_RESOLUTION, x=None, y=None, updates=None, data=None): self.resolution = resolution - self.x = None - self.y = None + self.x = x + self.y = y self.updates = updates self.data = data self.unfinished = False @@ -34,10 +36,11 @@ class MapHistory: try: with open(filename, 'rb') as f: resolution, x, y, width, height, num_updates = struct.unpack(' self.x+orig_width or maxy > self.y+orig_height: new_x, new_y = min(minx, self.x), min(miny, self.y) - new_width = min(maxx, self.x+orig_width)-new_x - new_height = min(maxy, self.y+orig_height)-new_y + new_width = max(maxx, self.x+orig_width)-new_x + new_height = max(maxy, self.y+orig_height)-new_y new_data = np.zeros((new_height, new_width), dtype=np.uint16) - dx, dy = new_x-self.x, new_y-self.y - new_data[dy:dx, (dy+orig_height):(dx+orig_width)] = data + dx, dy = self.x-new_x, self.y-new_y + new_data[dy:(dy+orig_height), dx:(dx+orig_width)] = data data = new_data self.x, self.y = new_x, new_y new_val = len(self.updates) for iy, y in enumerate(range(miny*res, maxy*res, res), start=miny-self.y): - for ix, x in enumerate(range(miny*res, maxy*res, res), start=minx-self.x): + for ix, x in enumerate(range(minx*res, maxx*res, res), start=minx-self.x): if prep.intersects(box(x, y, x+res, y+res)): + # print(iy, ix) data[iy, ix] = new_val + self.data = data self.unfinished = True def finish(self, cache_key): @@ -106,6 +111,28 @@ class GeometryChangeTracker: self._geometries_by_level = {} self._deleted_levels = set() + @staticmethod + def _level_filename(level_id): + return os.path.join(settings.CACHE_ROOT, 'level_base_%s' % level_id) + + def save(self, last_update, new_update): + for level_id in self._deleted_levels: + try: + os.remove(self._level_filename(level_id)) + except FileNotFoundError: + pass + self._geometries_by_level.pop(level_id, None) + + for level_id, geometries in self._geometries_by_level.items(): + geometries = unary_union(geometries) + if geometries.is_empty: + continue + history = MapHistory.open(self._level_filename(level_id), last_update) + history.add_new(geometries.buffer(1)) + history.finish(new_update) + history.save(self._level_filename(level_id)) + self.reset() + changed_geometries = GeometryChangeTracker() diff --git a/src/c3nav/mapdata/models/geometry/base.py b/src/c3nav/mapdata/models/geometry/base.py index 501f56cb..0b182d9b 100644 --- a/src/c3nav/mapdata/models/geometry/base.py +++ b/src/c3nav/mapdata/models/geometry/base.py @@ -82,8 +82,17 @@ class GeometryMixin(SerializableMixin): @property def geometry_changed(self): - return self.orig_geometry is None or (self.geometry is not self.orig_geometry and - not self.geometry.almost_equals(self.orig_geometry, 2)) + if self.orig_geometry is None: + return True + if self.geometry is self.orig_geometry: + return False + if not self.geometry.almost_equals(self.orig_geometry, 1): + return True + field = self._meta.get_field('geometry') + rounded = field.to_python(field.get_prep_value(self.geometry)) + if not rounded.almost_equals(self.orig_geometry, 2): + return True + return False def get_changed_geometry(self): return self.geometry if self.orig_geometry is None else self.geometry.symmetric_difference(self.orig_geometry) diff --git a/src/c3nav/mapdata/models/update.py b/src/c3nav/mapdata/models/update.py index 409273b9..6c4bc8d7 100644 --- a/src/c3nav/mapdata/models/update.py +++ b/src/c3nav/mapdata/models/update.py @@ -53,12 +53,18 @@ class MapUpdate(models.Model): if self.pk is not None: raise TypeError + old_cache_key = MapUpdate.current_cache_key() + from c3nav.mapdata.models import AltitudeArea - from c3nav.mapdata.render.cache import GeometryChangeTracker - from c3nav.mapdata.render.base import LevelRenderData AltitudeArea.recalculate() - GeometryChangeTracker() + + from c3nav.mapdata.render.base import LevelRenderData LevelRenderData.rebuild() + super().save(**kwargs) + + from c3nav.mapdata.cache import changed_geometries + changed_geometries.save(old_cache_key, self.cache_key) + cache.set('mapdata:last_update', (self.pk, self.datetime), 900) delete_old_cached_tiles.apply_async(countdown=5) diff --git a/src/c3nav/settings.py b/src/c3nav/settings.py index 76d426b9..6c44d37f 100644 --- a/src/c3nav/settings.py +++ b/src/c3nav/settings.py @@ -22,6 +22,7 @@ SOURCES_ROOT = os.path.join(DATA_DIR, 'sources') MAP_ROOT = os.path.join(DATA_DIR, 'map') RENDER_ROOT = os.path.join(DATA_DIR, 'render') TILES_ROOT = os.path.join(DATA_DIR, 'tiles') +CACHE_ROOT = os.path.join(DATA_DIR, 'cache') if not os.path.exists(DATA_DIR): os.mkdir(DATA_DIR) @@ -37,6 +38,8 @@ if not os.path.exists(RENDER_ROOT): os.mkdir(RENDER_ROOT) if not os.path.exists(TILES_ROOT): os.mkdir(TILES_ROOT) +if not os.path.exists(CACHE_ROOT): + os.mkdir(CACHE_ROOT) if config.has_option('django', 'secret'): SECRET_KEY = config.get('django', 'secret')