write binary map history index

src/c3nav/editor/models/changeset.py

@@ -20,6 +20,7 @@ from django.utils.translation import ungettext_lazy
 from c3nav.editor.models.changedobject import ApplyToInstanceError, ChangedObject
 from c3nav.editor.utils import is_created_pk
 from c3nav.editor.wrappers import ModelInstanceWrapper, ModelWrapper
+from c3nav.mapdata.cache import changed_geometries
 from c3nav.mapdata.models import LocationSlug, MapUpdate
 from c3nav.mapdata.models.locations import LocationRedirect
 from c3nav.mapdata.utils.models import get_submodels
@@ -453,11 +454,12 @@ class ChangeSet(models.Model):
                     changeset.save()
             elif self.direct_editing:
                 with MapUpdate.lock():
+                    changed_geometries.reset()
                     queries_before = len(connection.queries)
                     yield self
                     if any((q['sql'].startswith('UPDATE') or q['sql'].startswith('INSERT') or
                             q['sql'].startswith('DELETE')) for q in connection.queries[queries_before:]):
-                        MapUpdate.objects.create(user=user, type='direct_edit')
+                        update = MapUpdate.objects.create(user=user, type='direct_edit')
             else:
                 yield self
 

src/c3nav/mapdata/cache.py

@@ -1,4 +1,5 @@
 import math
+import os
 import struct
 
 import numpy as np
@@ -6,6 +7,7 @@ from django.conf import settings
 from django.db.models.signals import m2m_changed, post_delete
 from shapely import prepared
 from shapely.geometry import box
+from shapely.ops import unary_union
 
 from c3nav.mapdata.utils.models import get_submodels
 
@@ -23,8 +25,8 @@ class MapHistory:
 
     def __init__(self, resolution=settings.CACHE_RESOLUTION, x=None, y=None, updates=None, data=None):
         self.resolution = resolution
-        self.x = None
+        self.x = x
-        self.y = None
+        self.y = y
         self.updates = updates
         self.data = data
         self.unfinished = False
@@ -34,10 +36,11 @@ class MapHistory:
         try:
             with open(filename, 'rb') as f:
                 resolution, x, y, width, height, num_updates = struct.unpack('<BHHHHH', f.read(11))
-                updates = list(struct.unpack('16s'*num_updates, f.read(num_updates*16)))
+                updates = [s.decode().rstrip('\x00') for s in struct.unpack('16s'*num_updates, f.read(num_updates*16))]
-                data = np.frombuffer(f.read(width*height*2), np.uint16).reshape((height, width))
+                # noinspection PyTypeChecker
+                data = np.fromstring(f.read(width*height*2), np.uint16).reshape((height, width))
                 return cls(resolution, x, y, list(updates), data)
-        except FileNotFoundError:
+        except (FileNotFoundError, struct.error):
             if default_update is None:
                 raise
             return cls(updates=[default_update])
@@ -46,7 +49,7 @@ class MapHistory:
         with open(filename, 'wb') as f:
             f.write(struct.pack('<BHHHHH', self.resolution, self.x, self.y, *reversed(self.data.shape),
                                 len(self.updates)))
-            f.write(struct.pack('16s'*len(self.updates), *self.updates))
+            f.write(struct.pack('16s'*len(self.updates), *(s.encode() for s in self.updates)))
             f.write(self.data.tobytes('C'))
 
     def add_new(self, geometry):
@@ -70,20 +73,22 @@ class MapHistory:
             orig_height, orig_width = data.shape
             if minx < self.x or miny < self.y or maxx > self.x+orig_width or maxy > self.y+orig_height:
                 new_x, new_y = min(minx, self.x), min(miny, self.y)
-                new_width = min(maxx, self.x+orig_width)-new_x
+                new_width = max(maxx, self.x+orig_width)-new_x
-                new_height = min(maxy, self.y+orig_height)-new_y
+                new_height = max(maxy, self.y+orig_height)-new_y
                 new_data = np.zeros((new_height, new_width), dtype=np.uint16)
-                dx, dy = new_x-self.x, new_y-self.y
+                dx, dy = self.x-new_x, self.y-new_y
-                new_data[dy:dx, (dy+orig_height):(dx+orig_width)] = data
+                new_data[dy:(dy+orig_height), dx:(dx+orig_width)] = data
                 data = new_data
                 self.x, self.y = new_x, new_y
 
         new_val = len(self.updates)
         for iy, y in enumerate(range(miny*res, maxy*res, res), start=miny-self.y):
-            for ix, x in enumerate(range(miny*res, maxy*res, res), start=minx-self.x):
+            for ix, x in enumerate(range(minx*res, maxx*res, res), start=minx-self.x):
                 if prep.intersects(box(x, y, x+res, y+res)):
+                    # print(iy, ix)
                     data[iy, ix] = new_val
 
+        self.data = data
         self.unfinished = True
 
     def finish(self, cache_key):
@@ -106,6 +111,28 @@ class GeometryChangeTracker:
         self._geometries_by_level = {}
         self._deleted_levels = set()
 
+    @staticmethod
+    def _level_filename(level_id):
+        return os.path.join(settings.CACHE_ROOT, 'level_base_%s' % level_id)
+
+    def save(self, last_update, new_update):
+        for level_id in self._deleted_levels:
+            try:
+                os.remove(self._level_filename(level_id))
+            except FileNotFoundError:
+                pass
+            self._geometries_by_level.pop(level_id, None)
+
+        for level_id, geometries in self._geometries_by_level.items():
+            geometries = unary_union(geometries)
+            if geometries.is_empty:
+                continue
+            history = MapHistory.open(self._level_filename(level_id), last_update)
+            history.add_new(geometries.buffer(1))
+            history.finish(new_update)
+            history.save(self._level_filename(level_id))
+        self.reset()
+
 
 changed_geometries = GeometryChangeTracker()
 

src/c3nav/mapdata/models/geometry/base.py

@@ -82,8 +82,17 @@ class GeometryMixin(SerializableMixin):
 
     @property
     def geometry_changed(self):
-        return self.orig_geometry is None or (self.geometry is not self.orig_geometry and
+        if self.orig_geometry is None:
-                                              not self.geometry.almost_equals(self.orig_geometry, 2))
+            return True
+        if self.geometry is self.orig_geometry:
+            return False
+        if not self.geometry.almost_equals(self.orig_geometry, 1):
+            return True
+        field = self._meta.get_field('geometry')
+        rounded = field.to_python(field.get_prep_value(self.geometry))
+        if not rounded.almost_equals(self.orig_geometry, 2):
+            return True
+        return False
 
     def get_changed_geometry(self):
         return self.geometry if self.orig_geometry is None else self.geometry.symmetric_difference(self.orig_geometry)

src/c3nav/mapdata/models/update.py

@@ -53,12 +53,18 @@ class MapUpdate(models.Model):
         if self.pk is not None:
             raise TypeError
 
+        old_cache_key = MapUpdate.current_cache_key()
+
         from c3nav.mapdata.models import AltitudeArea
-        from c3nav.mapdata.render.cache import GeometryChangeTracker
-        from c3nav.mapdata.render.base import LevelRenderData
         AltitudeArea.recalculate()
-        GeometryChangeTracker()
+
+        from c3nav.mapdata.render.base import LevelRenderData
         LevelRenderData.rebuild()
+
         super().save(**kwargs)
+
+        from c3nav.mapdata.cache import changed_geometries
+        changed_geometries.save(old_cache_key, self.cache_key)
+
         cache.set('mapdata:last_update', (self.pk, self.datetime), 900)
         delete_old_cached_tiles.apply_async(countdown=5)

src/c3nav/settings.py

@@ -22,6 +22,7 @@ SOURCES_ROOT = os.path.join(DATA_DIR, 'sources')
 MAP_ROOT = os.path.join(DATA_DIR, 'map')
 RENDER_ROOT = os.path.join(DATA_DIR, 'render')
 TILES_ROOT = os.path.join(DATA_DIR, 'tiles')
+CACHE_ROOT = os.path.join(DATA_DIR, 'cache')
 
 if not os.path.exists(DATA_DIR):
     os.mkdir(DATA_DIR)
@@ -37,6 +38,8 @@ if not os.path.exists(RENDER_ROOT):
     os.mkdir(RENDER_ROOT)
 if not os.path.exists(TILES_ROOT):
     os.mkdir(TILES_ROOT)
+if not os.path.exists(CACHE_ROOT):
+    os.mkdir(CACHE_ROOT)
 
 if config.has_option('django', 'secret'):
     SECRET_KEY = config.get('django', 'secret')