split up c3nav.maputils.cache
This commit is contained in:
Laura Klünder
parent
116590b88f
commit
708620db0a
6 changed files with 423 additions and 411 deletions
|
|
@ -5,5 +5,5 @@ class MapdataConfig(AppConfig):
|
|||
name = 'c3nav.mapdata'
|
||||
|
||||
def ready(self):
|
||||
from c3nav.mapdata.utils.cache import register_signals
|
||||
from c3nav.mapdata.utils.cache.changes import register_signals
|
||||
register_signals()
|
||||
|
|
|
|||
|
|
@ -1,410 +0,0 @@
|
|||
import math
|
||||
import os
|
||||
import struct
|
||||
import threading
|
||||
from itertools import chain
|
||||
|
||||
import numpy as np
|
||||
from django.conf import settings
|
||||
from django.db.models.signals import m2m_changed, post_delete
|
||||
from PIL import Image
|
||||
from shapely import prepared
|
||||
from shapely.geometry import box
|
||||
from shapely.geometry.base import BaseGeometry
|
||||
from shapely.ops import unary_union
|
||||
|
||||
from c3nav.mapdata.utils.models import get_submodels
|
||||
|
||||
|
||||
class GeometryIndexed:
|
||||
# binary format (everything little-endian):
|
||||
# 1 byte (uint8): variant id
|
||||
# 1 byte (uint8): resolution
|
||||
# 2 bytes (uint16): origin x
|
||||
# 2 bytes (uint16): origin y
|
||||
# 2 bytes (uint16): origin width
|
||||
# 2 bytes (uint16): origin height
|
||||
# (optional meta data, depending on subclass)
|
||||
# x bytes data, line after line. (cell size depends on subclass)
|
||||
dtype = np.uint16
|
||||
variant_id = 0
|
||||
|
||||
def __init__(self, resolution=settings.CACHE_RESOLUTION, x=0, y=0, data=None, filename=None):
|
||||
self.resolution = resolution
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.data = data if data is not None else self._get_empty_array()
|
||||
self.filename = filename
|
||||
|
||||
@classmethod
|
||||
def _get_empty_array(cls):
|
||||
return np.empty((0, 0), dtype=cls.dtype)
|
||||
|
||||
@classmethod
|
||||
def open(cls, filename):
|
||||
with open(filename, 'rb') as f:
|
||||
instance = cls.read(f)
|
||||
instance.filename = filename
|
||||
return instance
|
||||
|
||||
@classmethod
|
||||
def read(cls, f):
|
||||
variant_id, resolution, x, y, width, height = struct.unpack('<BBHHHH', f.read(10))
|
||||
if variant_id != cls.variant_id:
|
||||
raise ValueError('variant id does not match')
|
||||
|
||||
kwargs = {
|
||||
'resolution': resolution,
|
||||
'x': x,
|
||||
'y': y,
|
||||
}
|
||||
cls._read_metadata(f, kwargs)
|
||||
|
||||
# noinspection PyTypeChecker
|
||||
kwargs['data'] = np.fromstring(f.read(width*height*cls.dtype().itemsize), cls.dtype).reshape((height, width))
|
||||
return cls(**kwargs)
|
||||
|
||||
@classmethod
|
||||
def _read_metadata(cls, f, kwargs):
|
||||
pass
|
||||
|
||||
def save(self, filename=None):
|
||||
if filename is None:
|
||||
filename = self.filename
|
||||
if filename is None:
|
||||
raise ValueError('Missing filename.')
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
self.write(f)
|
||||
|
||||
def write(self, f):
|
||||
f.write(struct.pack('<BBHHHH', self.variant_id, self.resolution, self.x, self.y, *reversed(self.data.shape)))
|
||||
self._write_metadata(f)
|
||||
f.write(self.data.tobytes('C'))
|
||||
|
||||
def _write_metadata(cls, f):
|
||||
pass
|
||||
|
||||
def _get_geometry_bounds(self, geometry):
|
||||
minx, miny, maxx, maxy = geometry.bounds
|
||||
return (
|
||||
int(math.floor(minx / self.resolution)),
|
||||
int(math.floor(miny / self.resolution)),
|
||||
int(math.ceil(maxx / self.resolution)),
|
||||
int(math.ceil(maxy / self.resolution)),
|
||||
)
|
||||
|
||||
def fit_bounds(self, minx, miny, maxx, maxy):
|
||||
height, width = self.data.shape
|
||||
|
||||
if self.data.size:
|
||||
minx = min(self.x, minx)
|
||||
miny = min(self.y, miny)
|
||||
maxx = max(self.x + width, maxx)
|
||||
maxy = max(self.y + height, maxy)
|
||||
|
||||
new_data = np.zeros((maxy - miny, maxx - minx), dtype=self.dtype)
|
||||
|
||||
if self.data.size:
|
||||
dx = self.x - minx
|
||||
dy = self.y - miny
|
||||
new_data[dy:(dy + height), dx:(dx + width)] = self.data
|
||||
|
||||
self.data = new_data
|
||||
self.x = minx
|
||||
self.y = miny
|
||||
|
||||
def get_geometry_cells(self, geometry, bounds=None):
|
||||
if bounds is None:
|
||||
bounds = self._get_geometry_bounds(geometry)
|
||||
minx, miny, maxx, maxy = bounds
|
||||
|
||||
height, width = self.data.shape
|
||||
minx = max(minx, self.x)
|
||||
miny = max(miny, self.y)
|
||||
maxx = min(maxx, self.x + width)
|
||||
maxy = min(maxy, self.y + height)
|
||||
|
||||
cells = np.zeros_like(self.data, dtype=np.bool)
|
||||
prep = prepared.prep(geometry)
|
||||
res = self.resolution
|
||||
for iy, y in enumerate(range(miny * res, maxy * res, res), start=miny - self.y):
|
||||
for ix, x in enumerate(range(minx * res, maxx * res, res), start=minx - self.x):
|
||||
if prep.intersects(box(x, y, x + res, y + res)):
|
||||
cells[iy, ix] = True
|
||||
|
||||
return cells
|
||||
|
||||
@property
|
||||
def bounds(self):
|
||||
height, width = self.data.shape
|
||||
return self.x, self.y, self.x+width, self.y+height
|
||||
|
||||
def __getitem__(self, key):
|
||||
if isinstance(key, BaseGeometry):
|
||||
bounds = self._get_geometry_bounds(key)
|
||||
return self.data[self.get_geometry_cells(key, bounds)]
|
||||
|
||||
if isinstance(key, tuple):
|
||||
xx, yy = key
|
||||
|
||||
minx = int(math.floor(xx.start / self.resolution))
|
||||
miny = int(math.floor(yy.start / self.resolution))
|
||||
maxx = int(math.ceil(xx.stop / self.resolution))
|
||||
maxy = int(math.ceil(yy.stop / self.resolution))
|
||||
|
||||
height, width = self.data.shape
|
||||
minx = min(self.x, minx) - self.x
|
||||
miny = min(self.y, miny) - self.x
|
||||
maxx = max(self.x + width, maxx) - self.y
|
||||
maxy = max(self.y + height, maxy) - self.y
|
||||
|
||||
return self.data[miny:maxy, minx:maxx].flatten()
|
||||
|
||||
raise TypeError('GeometryIndexed index must be a shapely geometry or tuple, not %s' % type(key).__name__)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if isinstance(key, BaseGeometry):
|
||||
bounds = self._get_geometry_bounds(key)
|
||||
self.fit_bounds(*bounds)
|
||||
cells = self.get_geometry_cells(key, bounds)
|
||||
self.data[cells] = value
|
||||
return
|
||||
|
||||
raise TypeError('GeometryIndexed index must be a shapely geometry, not %s' % type(key).__name__)
|
||||
|
||||
def to_image(self):
|
||||
from c3nav.mapdata.models import Source
|
||||
(minx, miny), (maxx, maxy) = Source.max_bounds()
|
||||
|
||||
height, width = self.data.shape
|
||||
image_data = np.zeros((int(math.ceil((maxy-miny)/self.resolution)),
|
||||
int(math.ceil((maxx-minx)/self.resolution))), dtype=np.uint8)
|
||||
|
||||
if self.data.size:
|
||||
minval = min(self.data.min(), 0)
|
||||
maxval = max(self.data.max(), minval+0.01)
|
||||
visible_data = ((self.data.astype(float)-minval)*255/(maxval-minval)).clip(0, 255).astype(np.uint8)
|
||||
image_data[self.y:self.y+height, self.x:self.x+width] = visible_data
|
||||
|
||||
return Image.fromarray(np.flip(image_data, axis=0), 'L')
|
||||
|
||||
|
||||
class MapHistory(GeometryIndexed):
|
||||
# metadata format:
|
||||
# 2 bytes (uint16): number of updates
|
||||
# n uptates times:
|
||||
# 4 bytes (uint32): update id
|
||||
# 4 bytes (uint32): timestamp
|
||||
dtype = np.uint16
|
||||
variant_id = 1
|
||||
|
||||
def __init__(self, updates, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.updates = updates
|
||||
|
||||
@classmethod
|
||||
def _read_metadata(cls, f, kwargs):
|
||||
num_updates = struct.unpack('<H', f.read(2))[0]
|
||||
updates = struct.unpack('<'+'II'*num_updates, f.read(num_updates*8))
|
||||
updates = list(zip(updates[0::2], updates[1::2]))
|
||||
kwargs['updates'] = updates
|
||||
|
||||
def _write_metadata(self, f):
|
||||
f.write(struct.pack('<H', len(self.updates)))
|
||||
f.write(struct.pack('<'+'II'*len(self.updates), *chain(*self.updates)))
|
||||
|
||||
@classmethod
|
||||
def open(cls, filename, default_update=None):
|
||||
try:
|
||||
instance = super().open(filename)
|
||||
except FileNotFoundError:
|
||||
if default_update is None:
|
||||
from c3nav.mapdata.models import MapUpdate
|
||||
default_update = MapUpdate.last_processed_update()
|
||||
instance = cls(updates=[default_update], filename=filename)
|
||||
instance.save()
|
||||
return instance
|
||||
|
||||
@staticmethod
|
||||
def level_filename(level_id, mode):
|
||||
return os.path.join(settings.CACHE_ROOT, 'level_%d_history_%s' % (level_id, mode))
|
||||
|
||||
@classmethod
|
||||
def open_level(cls, level_id, mode, default_update=None):
|
||||
return cls.open(cls.level_filename(level_id, mode), default_update)
|
||||
|
||||
cached = {}
|
||||
cache_key = None
|
||||
cache_lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def open_level_cached(cls, level_id, mode):
|
||||
with cls.cache_lock:
|
||||
from c3nav.mapdata.models import MapUpdate
|
||||
cache_key = MapUpdate.current_processed_cache_key()
|
||||
if cls.cache_key != cache_key:
|
||||
cls.cache_key = cache_key
|
||||
cls.cached = {}
|
||||
else:
|
||||
result = cls.cached.get((level_id, mode), None)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
result = cls.open_level(level_id, mode)
|
||||
cls.cached[(level_id, mode)] = result
|
||||
return result
|
||||
|
||||
def add_geometry(self, geometry, update):
|
||||
if self.updates[-1] != update:
|
||||
self.updates.append(update)
|
||||
|
||||
self[geometry] = len(self.updates) - 1
|
||||
|
||||
def simplify(self):
|
||||
# remove updates that have no longer any array cells
|
||||
new_updates = ((i, update, (self.data == i)) for i, update in enumerate(self.updates))
|
||||
self.updates, new_affected = zip(*((update, affected) for i, update, affected in new_updates
|
||||
if i == 0 or affected.any()))
|
||||
for i, affected in enumerate(new_affected):
|
||||
self.data[affected] = i
|
||||
|
||||
def write(self, *args, **kwargs):
|
||||
self.simplify()
|
||||
super().write(*args, **kwargs)
|
||||
|
||||
def composite(self, other, mask_geometry):
|
||||
if self.resolution != other.resolution:
|
||||
raise ValueError('Cannot composite with different resolutions.')
|
||||
|
||||
self.fit_bounds(*other.bounds)
|
||||
other.fit_bounds(*self.bounds)
|
||||
|
||||
# merge update lists
|
||||
self_update_i = {update: i for i, update in enumerate(self.updates)}
|
||||
other_update_i = {update: i for i, update in enumerate(other.updates)}
|
||||
new_updates = sorted(set(self_update_i.keys()) | set(other_update_i.keys()))
|
||||
|
||||
# reindex according to merged update list
|
||||
self_data = self.data.copy()
|
||||
other_data = other.data.copy()
|
||||
for i, update in enumerate(new_updates):
|
||||
if update in self_update_i:
|
||||
self.data[self_data == self_update_i[update]] = i
|
||||
if update in other_update_i:
|
||||
other_data[other_data == other_update_i[update]] = i
|
||||
|
||||
# calculate maximum
|
||||
maximum = np.maximum(self.data, other_data)
|
||||
|
||||
# add with mask
|
||||
if mask_geometry is not None:
|
||||
mask = self.get_geometry_cells(mask_geometry)
|
||||
self.data[mask] = maximum[mask]
|
||||
else:
|
||||
self.data = maximum
|
||||
|
||||
# write new updates
|
||||
self.updates = new_updates
|
||||
self.simplify()
|
||||
|
||||
def last_update(self, minx, miny, maxx, maxy):
|
||||
cells = self[minx:maxx, miny:maxy]
|
||||
if cells.size:
|
||||
return self.updates[cells.max()]
|
||||
return self.updates[0]
|
||||
|
||||
|
||||
class GeometryChangeTracker:
|
||||
def __init__(self):
|
||||
self._geometries_by_level = {}
|
||||
self._deleted_levels = set()
|
||||
self._unary_unions = {}
|
||||
|
||||
def register(self, level_id, geometry):
|
||||
self._geometries_by_level.setdefault(level_id, []).append(geometry.buffer(0.01))
|
||||
self._unary_unions.pop(level_id, None)
|
||||
|
||||
def level_deleted(self, level_id):
|
||||
self._deleted_levels.add(level_id)
|
||||
self._unary_unions.pop(level_id, None)
|
||||
|
||||
def reset(self):
|
||||
self._geometries_by_level = {}
|
||||
self._deleted_levels = set()
|
||||
self._unary_unions = {}
|
||||
|
||||
def _get_unary_union(self, level_id):
|
||||
union = self._unary_unions.get(level_id)
|
||||
if union is None:
|
||||
union = unary_union(self._geometries_by_level[level_id])
|
||||
self._unary_unions[level_id] = union
|
||||
return union
|
||||
|
||||
@property
|
||||
def area(self):
|
||||
return sum((self._get_unary_union(level_id).area
|
||||
for level_id in self._geometries_by_level.keys()
|
||||
if level_id not in self._deleted_levels), 0)
|
||||
|
||||
def finalize(self):
|
||||
for level_id in self._deleted_levels:
|
||||
try:
|
||||
os.remove(MapHistory.level_filename(level_id, mode='base'))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
self._geometries_by_level.pop(level_id, None)
|
||||
self._deleted_levels = set()
|
||||
|
||||
def combine(self, other):
|
||||
self.finalize()
|
||||
other.finalize()
|
||||
for level_id in other._geometries_by_level.keys():
|
||||
self._geometries_by_level.setdefault(level_id, []).append(other._get_unary_union(level_id))
|
||||
self._unary_unions = {}
|
||||
|
||||
def save(self, last_update, new_update):
|
||||
self.finalize()
|
||||
|
||||
for level_id, geometries in self._geometries_by_level.items():
|
||||
geometries = unary_union(geometries)
|
||||
if geometries.is_empty:
|
||||
continue
|
||||
history = MapHistory.open_level(level_id, mode='base', default_update=last_update)
|
||||
history.add_geometry(geometries.buffer(1), new_update)
|
||||
history.save()
|
||||
self.reset()
|
||||
|
||||
|
||||
changed_geometries = GeometryChangeTracker()
|
||||
|
||||
|
||||
def geometry_deleted(sender, instance, **kwargs):
|
||||
instance.register_delete()
|
||||
|
||||
|
||||
def locationgroup_changed(sender, instance, action, reverse, model, pk_set, using, **kwargs):
|
||||
if action not in ('post_add', 'post_remove', 'post_clear'):
|
||||
return
|
||||
|
||||
if not reverse:
|
||||
instance.register_change(force=True)
|
||||
else:
|
||||
if action not in 'post_clear':
|
||||
raise NotImplementedError
|
||||
query = model.objects.filter(pk__in=pk_set)
|
||||
from c3nav.mapdata.models.geometry.space import SpaceGeometryMixin
|
||||
if issubclass(model, SpaceGeometryMixin):
|
||||
query = query.select_related('space')
|
||||
for obj in query:
|
||||
obj.register_change(force=True)
|
||||
|
||||
|
||||
def register_signals():
|
||||
from c3nav.mapdata.models.geometry.base import GeometryMixin
|
||||
for model in get_submodels(GeometryMixin):
|
||||
post_delete.connect(geometry_deleted, sender=model)
|
||||
|
||||
from c3nav.mapdata.models.locations import SpecificLocation
|
||||
for model in get_submodels(SpecificLocation):
|
||||
m2m_changed.connect(locationgroup_changed, sender=model.groups.through)
|
||||
3
src/c3nav/mapdata/utils/cache/__init__.py
vendored
Normal file
3
src/c3nav/mapdata/utils/cache/__init__.py
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from c3nav.mapdata.utils.cache.indexed import GeometryIndexed # noqa
|
||||
from c3nav.mapdata.utils.cache.maphistory import MapHistory # noqa
|
||||
from c3nav.mapdata.utils.cache.changes import changed_geometries # noqa
|
||||
102
src/c3nav/mapdata/utils/cache/changes.py
vendored
Normal file
102
src/c3nav/mapdata/utils/cache/changes.py
vendored
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import os
|
||||
|
||||
from django.db.models.signals import post_delete, m2m_changed
|
||||
from shapely.ops import unary_union
|
||||
|
||||
from c3nav.mapdata.utils.cache.maphistory import MapHistory
|
||||
from c3nav.mapdata.utils.models import get_submodels
|
||||
|
||||
|
||||
class GeometryChangeTracker:
|
||||
def __init__(self):
|
||||
self._geometries_by_level = {}
|
||||
self._deleted_levels = set()
|
||||
self._unary_unions = {}
|
||||
|
||||
def register(self, level_id, geometry):
|
||||
self._geometries_by_level.setdefault(level_id, []).append(geometry.buffer(0.01))
|
||||
self._unary_unions.pop(level_id, None)
|
||||
|
||||
def level_deleted(self, level_id):
|
||||
self._deleted_levels.add(level_id)
|
||||
self._unary_unions.pop(level_id, None)
|
||||
|
||||
def reset(self):
|
||||
self._geometries_by_level = {}
|
||||
self._deleted_levels = set()
|
||||
self._unary_unions = {}
|
||||
|
||||
def _get_unary_union(self, level_id):
|
||||
union = self._unary_unions.get(level_id)
|
||||
if union is None:
|
||||
union = unary_union(self._geometries_by_level[level_id])
|
||||
self._unary_unions[level_id] = union
|
||||
return union
|
||||
|
||||
@property
|
||||
def area(self):
|
||||
return sum((self._get_unary_union(level_id).area
|
||||
for level_id in self._geometries_by_level.keys()
|
||||
if level_id not in self._deleted_levels), 0)
|
||||
|
||||
def finalize(self):
|
||||
for level_id in self._deleted_levels:
|
||||
try:
|
||||
os.remove(MapHistory.level_filename(level_id, mode='base'))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
self._geometries_by_level.pop(level_id, None)
|
||||
self._deleted_levels = set()
|
||||
|
||||
def combine(self, other):
|
||||
self.finalize()
|
||||
other.finalize()
|
||||
for level_id in other._geometries_by_level.keys():
|
||||
self._geometries_by_level.setdefault(level_id, []).append(other._get_unary_union(level_id))
|
||||
self._unary_unions = {}
|
||||
|
||||
def save(self, last_update, new_update):
|
||||
self.finalize()
|
||||
|
||||
for level_id, geometries in self._geometries_by_level.items():
|
||||
geometries = unary_union(geometries)
|
||||
if geometries.is_empty:
|
||||
continue
|
||||
history = MapHistory.open_level(level_id, mode='base', default_update=last_update)
|
||||
history.add_geometry(geometries.buffer(1), new_update)
|
||||
history.save()
|
||||
self.reset()
|
||||
|
||||
|
||||
changed_geometries = GeometryChangeTracker()
|
||||
|
||||
|
||||
def geometry_deleted(sender, instance, **kwargs):
|
||||
instance.register_delete()
|
||||
|
||||
|
||||
def locationgroup_changed(sender, instance, action, reverse, model, pk_set, using, **kwargs):
|
||||
if action not in ('post_add', 'post_remove', 'post_clear'):
|
||||
return
|
||||
|
||||
if not reverse:
|
||||
instance.register_change(force=True)
|
||||
else:
|
||||
if action not in 'post_clear':
|
||||
raise NotImplementedError
|
||||
query = model.objects.filter(pk__in=pk_set)
|
||||
from c3nav.mapdata.models.geometry.space import SpaceGeometryMixin
|
||||
if issubclass(model, SpaceGeometryMixin):
|
||||
query = query.select_related('space')
|
||||
for obj in query:
|
||||
obj.register_change(force=True)
|
||||
|
||||
|
||||
def register_signals():
|
||||
from c3nav.mapdata.models.geometry.base import GeometryMixin
|
||||
for model in get_submodels(GeometryMixin):
|
||||
post_delete.connect(geometry_deleted, sender=model)
|
||||
|
||||
from c3nav.mapdata.models.locations import SpecificLocation
|
||||
for model in get_submodels(SpecificLocation):
|
||||
m2m_changed.connect(locationgroup_changed, sender=model.groups.through)
|
||||
183
src/c3nav/mapdata/utils/cache/indexed.py
vendored
Normal file
183
src/c3nav/mapdata/utils/cache/indexed.py
vendored
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
import math
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from django.conf import settings
|
||||
from shapely import prepared
|
||||
from shapely.geometry import box
|
||||
from shapely.geometry.base import BaseGeometry
|
||||
|
||||
|
||||
class GeometryIndexed:
|
||||
# binary format (everything little-endian):
|
||||
# 1 byte (uint8): variant id
|
||||
# 1 byte (uint8): resolution
|
||||
# 2 bytes (uint16): origin x
|
||||
# 2 bytes (uint16): origin y
|
||||
# 2 bytes (uint16): origin width
|
||||
# 2 bytes (uint16): origin height
|
||||
# (optional meta data, depending on subclass)
|
||||
# x bytes data, line after line. (cell size depends on subclass)
|
||||
dtype = np.uint16
|
||||
variant_id = 0
|
||||
|
||||
def __init__(self, resolution=settings.CACHE_RESOLUTION, x=0, y=0, data=None, filename=None):
|
||||
self.resolution = resolution
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.data = data if data is not None else self._get_empty_array()
|
||||
self.filename = filename
|
||||
|
||||
@classmethod
|
||||
def _get_empty_array(cls):
|
||||
return np.empty((0, 0), dtype=cls.dtype)
|
||||
|
||||
@classmethod
|
||||
def open(cls, filename):
|
||||
with open(filename, 'rb') as f:
|
||||
instance = cls.read(f)
|
||||
instance.filename = filename
|
||||
return instance
|
||||
|
||||
@classmethod
|
||||
def read(cls, f):
|
||||
variant_id, resolution, x, y, width, height = struct.unpack('<BBHHHH', f.read(10))
|
||||
if variant_id != cls.variant_id:
|
||||
raise ValueError('variant id does not match')
|
||||
|
||||
kwargs = {
|
||||
'resolution': resolution,
|
||||
'x': x,
|
||||
'y': y,
|
||||
}
|
||||
cls._read_metadata(f, kwargs)
|
||||
|
||||
# noinspection PyTypeChecker
|
||||
kwargs['data'] = np.fromstring(f.read(width*height*cls.dtype().itemsize), cls.dtype).reshape((height, width))
|
||||
return cls(**kwargs)
|
||||
|
||||
@classmethod
|
||||
def _read_metadata(cls, f, kwargs):
|
||||
pass
|
||||
|
||||
def save(self, filename=None):
|
||||
if filename is None:
|
||||
filename = self.filename
|
||||
if filename is None:
|
||||
raise ValueError('Missing filename.')
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
self.write(f)
|
||||
|
||||
def write(self, f):
|
||||
f.write(struct.pack('<BBHHHH', self.variant_id, self.resolution, self.x, self.y, *reversed(self.data.shape)))
|
||||
self._write_metadata(f)
|
||||
f.write(self.data.tobytes('C'))
|
||||
|
||||
def _write_metadata(cls, f):
|
||||
pass
|
||||
|
||||
def _get_geometry_bounds(self, geometry):
|
||||
minx, miny, maxx, maxy = geometry.bounds
|
||||
return (
|
||||
int(math.floor(minx / self.resolution)),
|
||||
int(math.floor(miny / self.resolution)),
|
||||
int(math.ceil(maxx / self.resolution)),
|
||||
int(math.ceil(maxy / self.resolution)),
|
||||
)
|
||||
|
||||
def fit_bounds(self, minx, miny, maxx, maxy):
|
||||
height, width = self.data.shape
|
||||
|
||||
if self.data.size:
|
||||
minx = min(self.x, minx)
|
||||
miny = min(self.y, miny)
|
||||
maxx = max(self.x + width, maxx)
|
||||
maxy = max(self.y + height, maxy)
|
||||
|
||||
new_data = np.zeros((maxy - miny, maxx - minx), dtype=self.dtype)
|
||||
|
||||
if self.data.size:
|
||||
dx = self.x - minx
|
||||
dy = self.y - miny
|
||||
new_data[dy:(dy + height), dx:(dx + width)] = self.data
|
||||
|
||||
self.data = new_data
|
||||
self.x = minx
|
||||
self.y = miny
|
||||
|
||||
def get_geometry_cells(self, geometry, bounds=None):
|
||||
if bounds is None:
|
||||
bounds = self._get_geometry_bounds(geometry)
|
||||
minx, miny, maxx, maxy = bounds
|
||||
|
||||
height, width = self.data.shape
|
||||
minx = max(minx, self.x)
|
||||
miny = max(miny, self.y)
|
||||
maxx = min(maxx, self.x + width)
|
||||
maxy = min(maxy, self.y + height)
|
||||
|
||||
cells = np.zeros_like(self.data, dtype=np.bool)
|
||||
prep = prepared.prep(geometry)
|
||||
res = self.resolution
|
||||
for iy, y in enumerate(range(miny * res, maxy * res, res), start=miny - self.y):
|
||||
for ix, x in enumerate(range(minx * res, maxx * res, res), start=minx - self.x):
|
||||
if prep.intersects(box(x, y, x + res, y + res)):
|
||||
cells[iy, ix] = True
|
||||
|
||||
return cells
|
||||
|
||||
@property
|
||||
def bounds(self):
|
||||
height, width = self.data.shape
|
||||
return self.x, self.y, self.x+width, self.y+height
|
||||
|
||||
def __getitem__(self, key):
|
||||
if isinstance(key, BaseGeometry):
|
||||
bounds = self._get_geometry_bounds(key)
|
||||
return self.data[self.get_geometry_cells(key, bounds)]
|
||||
|
||||
if isinstance(key, tuple):
|
||||
xx, yy = key
|
||||
|
||||
minx = int(math.floor(xx.start / self.resolution))
|
||||
miny = int(math.floor(yy.start / self.resolution))
|
||||
maxx = int(math.ceil(xx.stop / self.resolution))
|
||||
maxy = int(math.ceil(yy.stop / self.resolution))
|
||||
|
||||
height, width = self.data.shape
|
||||
minx = min(self.x, minx) - self.x
|
||||
miny = min(self.y, miny) - self.x
|
||||
maxx = max(self.x + width, maxx) - self.y
|
||||
maxy = max(self.y + height, maxy) - self.y
|
||||
|
||||
return self.data[miny:maxy, minx:maxx].flatten()
|
||||
|
||||
raise TypeError('GeometryIndexed index must be a shapely geometry or tuple, not %s' % type(key).__name__)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if isinstance(key, BaseGeometry):
|
||||
bounds = self._get_geometry_bounds(key)
|
||||
self.fit_bounds(*bounds)
|
||||
cells = self.get_geometry_cells(key, bounds)
|
||||
self.data[cells] = value
|
||||
return
|
||||
|
||||
raise TypeError('GeometryIndexed index must be a shapely geometry, not %s' % type(key).__name__)
|
||||
|
||||
def to_image(self):
|
||||
from c3nav.mapdata.models import Source
|
||||
(minx, miny), (maxx, maxy) = Source.max_bounds()
|
||||
|
||||
height, width = self.data.shape
|
||||
image_data = np.zeros((int(math.ceil((maxy-miny)/self.resolution)),
|
||||
int(math.ceil((maxx-minx)/self.resolution))), dtype=np.uint8)
|
||||
|
||||
if self.data.size:
|
||||
minval = min(self.data.min(), 0)
|
||||
maxval = max(self.data.max(), minval+0.01)
|
||||
visible_data = ((self.data.astype(float)-minval)*255/(maxval-minval)).clip(0, 255).astype(np.uint8)
|
||||
image_data[self.y:self.y+height, self.x:self.x+width] = visible_data
|
||||
|
||||
return Image.fromarray(np.flip(image_data, axis=0), 'L')
|
||||
134
src/c3nav/mapdata/utils/cache/maphistory.py
vendored
Normal file
134
src/c3nav/mapdata/utils/cache/maphistory.py
vendored
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import os
|
||||
import struct
|
||||
import threading
|
||||
from itertools import chain
|
||||
|
||||
import numpy as np
|
||||
from django.conf import settings
|
||||
|
||||
from c3nav.mapdata.utils.cache import GeometryIndexed
|
||||
|
||||
|
||||
class MapHistory(GeometryIndexed):
|
||||
# metadata format:
|
||||
# 2 bytes (uint16): number of updates
|
||||
# n uptates times:
|
||||
# 4 bytes (uint32): update id
|
||||
# 4 bytes (uint32): timestamp
|
||||
dtype = np.uint16
|
||||
variant_id = 1
|
||||
|
||||
def __init__(self, updates, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.updates = updates
|
||||
|
||||
@classmethod
|
||||
def _read_metadata(cls, f, kwargs):
|
||||
num_updates = struct.unpack('<H', f.read(2))[0]
|
||||
updates = struct.unpack('<'+'II'*num_updates, f.read(num_updates*8))
|
||||
updates = list(zip(updates[0::2], updates[1::2]))
|
||||
kwargs['updates'] = updates
|
||||
|
||||
def _write_metadata(self, f):
|
||||
f.write(struct.pack('<H', len(self.updates)))
|
||||
f.write(struct.pack('<'+'II'*len(self.updates), *chain(*self.updates)))
|
||||
|
||||
@classmethod
|
||||
def open(cls, filename, default_update=None):
|
||||
try:
|
||||
instance = super().open(filename)
|
||||
except FileNotFoundError:
|
||||
if default_update is None:
|
||||
from c3nav.mapdata.models import MapUpdate
|
||||
default_update = MapUpdate.last_processed_update()
|
||||
instance = cls(updates=[default_update], filename=filename)
|
||||
instance.save()
|
||||
return instance
|
||||
|
||||
@staticmethod
|
||||
def level_filename(level_id, mode):
|
||||
return os.path.join(settings.CACHE_ROOT, 'level_%d_history_%s' % (level_id, mode))
|
||||
|
||||
@classmethod
|
||||
def open_level(cls, level_id, mode, default_update=None):
|
||||
return cls.open(cls.level_filename(level_id, mode), default_update)
|
||||
|
||||
cached = {}
|
||||
cache_key = None
|
||||
cache_lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def open_level_cached(cls, level_id, mode):
|
||||
with cls.cache_lock:
|
||||
from c3nav.mapdata.models import MapUpdate
|
||||
cache_key = MapUpdate.current_processed_cache_key()
|
||||
if cls.cache_key != cache_key:
|
||||
cls.cache_key = cache_key
|
||||
cls.cached = {}
|
||||
else:
|
||||
result = cls.cached.get((level_id, mode), None)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
result = cls.open_level(level_id, mode)
|
||||
cls.cached[(level_id, mode)] = result
|
||||
return result
|
||||
|
||||
def add_geometry(self, geometry, update):
|
||||
if self.updates[-1] != update:
|
||||
self.updates.append(update)
|
||||
|
||||
self[geometry] = len(self.updates) - 1
|
||||
|
||||
def simplify(self):
|
||||
# remove updates that have no longer any array cells
|
||||
new_updates = ((i, update, (self.data == i)) for i, update in enumerate(self.updates))
|
||||
self.updates, new_affected = zip(*((update, affected) for i, update, affected in new_updates
|
||||
if i == 0 or affected.any()))
|
||||
for i, affected in enumerate(new_affected):
|
||||
self.data[affected] = i
|
||||
|
||||
def write(self, *args, **kwargs):
|
||||
self.simplify()
|
||||
super().write(*args, **kwargs)
|
||||
|
||||
def composite(self, other, mask_geometry):
|
||||
if self.resolution != other.resolution:
|
||||
raise ValueError('Cannot composite with different resolutions.')
|
||||
|
||||
self.fit_bounds(*other.bounds)
|
||||
other.fit_bounds(*self.bounds)
|
||||
|
||||
# merge update lists
|
||||
self_update_i = {update: i for i, update in enumerate(self.updates)}
|
||||
other_update_i = {update: i for i, update in enumerate(other.updates)}
|
||||
new_updates = sorted(set(self_update_i.keys()) | set(other_update_i.keys()))
|
||||
|
||||
# reindex according to merged update list
|
||||
self_data = self.data.copy()
|
||||
other_data = other.data.copy()
|
||||
for i, update in enumerate(new_updates):
|
||||
if update in self_update_i:
|
||||
self.data[self_data == self_update_i[update]] = i
|
||||
if update in other_update_i:
|
||||
other_data[other_data == other_update_i[update]] = i
|
||||
|
||||
# calculate maximum
|
||||
maximum = np.maximum(self.data, other_data)
|
||||
|
||||
# add with mask
|
||||
if mask_geometry is not None:
|
||||
mask = self.get_geometry_cells(mask_geometry)
|
||||
self.data[mask] = maximum[mask]
|
||||
else:
|
||||
self.data = maximum
|
||||
|
||||
# write new updates
|
||||
self.updates = new_updates
|
||||
self.simplify()
|
||||
|
||||
def last_update(self, minx, miny, maxx, maxy):
|
||||
cells = self[minx:maxx, miny:maxy]
|
||||
if cells.size:
|
||||
return self.updates[cells.max()]
|
||||
return self.updates[0]
|
||||
Loading…
Add table
Add a link
Reference in a new issue