diff --git a/src/c3nav/mapdata/cache.py b/src/c3nav/mapdata/cache.py index dae32adb..8190ffc5 100644 --- a/src/c3nav/mapdata/cache.py +++ b/src/c3nav/mapdata/cache.py @@ -5,6 +5,7 @@ from itertools import chain import numpy as np from django.conf import settings +from django.core.cache import cache from django.db.models.signals import m2m_changed, post_delete from PIL import Image from shapely import prepared @@ -64,6 +65,17 @@ class MapHistory: def open_level(cls, level_id, mode, default_update=None): return cls.open(cls.level_filename(level_id, mode), default_update) + @classmethod + def open_level_cached(cls, level_id, mode, cache_key=None): + if cache_key is None: + cache_key = MapUpdate.current_cache_key() + cache_key = 'mapdata:map-history-%d-%s:%s' % (level_id, mode, cache_key) + result = cache.get(cache_key, None) + if result is None: + result = cls.open_level(level_id, mode) + cache.set(cache_key, result, 120) + return result + def save(self, filename=None): if filename is None: filename = self.filename @@ -205,6 +217,17 @@ class MapHistory: return Image.fromarray(np.flip(image_data, axis=0), 'L') + def last_update(self, minx, miny, maxx, maxy): + res = self.resolution + height, width = self.data.shape + minx = max(int(math.floor(minx/res)), self.x)-self.x + miny = max(int(math.floor(miny/res)), self.y)-self.y + maxx = min(int(math.ceil(maxx/res)), self.x+width)-self.x + maxy = min(int(math.ceil(maxy/res)), self.y+height)-self.y + if minx >= maxx or miny >= maxy: + return self.updates[0] + return self.updates[self.data[miny:maxy, minx:maxx].max()] + class GeometryChangeTracker: def __init__(self): diff --git a/src/c3nav/mapdata/models/update.py b/src/c3nav/mapdata/models/update.py index 866f9fa3..189e0435 100644 --- a/src/c3nav/mapdata/models/update.py +++ b/src/c3nav/mapdata/models/update.py @@ -41,11 +41,14 @@ class MapUpdate(models.Model): @property def cache_key(self): - return int_to_base36(self.pk)+'_'+int_to_base36(int(make_naive(self.datetime).timestamp())) + return self.build_cache_key(self.pk, int(make_naive(self.datetime).timestamp())) @classmethod def current_cache_key(cls): - pk, timestamp = cls.last_update() + return cls.build_cache_key(*cls.last_update()) + + @staticmethod + def build_cache_key(pk, timestamp): return int_to_base36(pk)+'_'+int_to_base36(timestamp) @classmethod diff --git a/src/c3nav/mapdata/render/base.py b/src/c3nav/mapdata/render/base.py index 7bdcdcef..68ef9ae4 100644 --- a/src/c3nav/mapdata/render/base.py +++ b/src/c3nav/mapdata/render/base.py @@ -8,6 +8,17 @@ from c3nav.mapdata.cache import MapHistory from c3nav.mapdata.models import Level, MapUpdate +def get_render_level_ids(cache_key=None): + if cache_key is None: + cache_key = MapUpdate.current_cache_key() + cache_key = 'mapdata:render-level-ids:'+cache_key + levels = cache.get(cache_key, None) + if levels is None: + levels = set(Level.objects.values_list('pk', flat=True)) + cache.set(cache_key, levels, 300) + return levels + + class AltitudeAreaGeometries: def __init__(self, altitudearea=None, colors=None): if altitudearea is not None: diff --git a/src/c3nav/mapdata/render/svg.py b/src/c3nav/mapdata/render/svg.py index f8573d04..471511b9 100644 --- a/src/c3nav/mapdata/render/svg.py +++ b/src/c3nav/mapdata/render/svg.py @@ -1,8 +1,11 @@ +from django.core.cache import cache from django.utils.functional import cached_property from shapely import prepared from shapely.geometry import box from shapely.ops import unary_union +from c3nav.mapdata.cache import MapHistory +from c3nav.mapdata.models import MapUpdate from c3nav.mapdata.render.base import get_level_render_data from c3nav.mapdata.utils.svg import SVGImage @@ -25,13 +28,25 @@ class SVGRenderer: def level_render_data(self): return get_level_render_data(self.level) - def check_level(self): - return self.level_render_data + @cached_property + def last_update(self): + return MapHistory.open_level_cached(self.level, 'render').last_update(self.minx, self.miny, + self.maxx, self.maxy) + + @cached_property + def update_cache_key(self): + return MapUpdate.build_cache_key(*self.last_update) @cached_property def affected_access_restrictions(self): - return set(ar for ar, area in self.level_render_data.access_restriction_affected.items() - if area.intersects(self.bbox)) + cache_key = 'mapdata:affected-ars-%.2f-%.2f-%.2f-%.2f:%s' % (self.minx, self.miny, self.maxx, self.maxy, + self.update_cache_key) + result = cache.get(cache_key, None) + if result is None: + result = set(ar for ar, area in self.level_render_data.access_restriction_affected.items() + if area.intersects(self.bbox)) + cache.set(cache_key, result, 120) + return result @cached_property def unlocked_access_restrictions(self): @@ -43,6 +58,10 @@ class SVGRenderer: def access_cache_key(self): return '_'.join(str(i) for i in sorted(self.unlocked_access_restrictions)) or '0' + @cached_property + def cache_key(self): + return self.update_cache_key + ':' + self.access_cache_key + def render(self): svg = SVGImage(bounds=((self.miny, self.minx), (self.maxy, self.maxx)), scale=self.scale, buffer=1) diff --git a/src/c3nav/mapdata/views.py b/src/c3nav/mapdata/views.py index 5c8be822..986e3256 100644 --- a/src/c3nav/mapdata/views.py +++ b/src/c3nav/mapdata/views.py @@ -1,6 +1,7 @@ import os from django.conf import settings +from django.core.cache import cache from django.core.exceptions import PermissionDenied from django.http import Http404, HttpResponse, HttpResponseNotModified from django.shortcuts import get_object_or_404 @@ -8,6 +9,7 @@ from shapely.geometry import box from c3nav.mapdata.cache import MapHistory from c3nav.mapdata.models import Level, MapUpdate, Source +from c3nav.mapdata.render.base import get_render_level_ids from c3nav.mapdata.render.svg import SVGRenderer @@ -21,8 +23,7 @@ def tile(request, level, zoom, x, y, format): if not (0 <= zoom <= 10): raise Http404 - bounds = Source.max_bounds() - + # calculate bounds x, y = int(x), int(y) size = 256/2**zoom minx = size * x @@ -30,37 +31,59 @@ def tile(request, level, zoom, x, y, format): maxx = minx + size maxy = miny + size + # error 404 if tiles is out of bounds + bounds = Source.max_bounds() if not box(bounds[0][1], bounds[0][0], bounds[1][1], bounds[1][0]).intersects(box(minx, miny, maxx, maxy)): raise Http404 - renderer = SVGRenderer(level, miny, minx, maxy, maxx, scale=2**zoom, user=request.user) + # is this a valid level? + cache_key = MapUpdate.current_cache_key() + level = int(level) + if level not in get_render_level_ids(cache_key): + raise Http404 - update_cache_key = MapUpdate.current_cache_key() - access_cache_key = renderer.access_cache_key - etag = update_cache_key+'_'+access_cache_key + # init renderer + renderer = SVGRenderer(level, miny, minx, maxy, maxx, scale=2 ** zoom, user=request.user) + tile_cache_key = renderer.cache_key + update_cache_key = renderer.update_cache_key + # check browser cache + etag = tile_cache_key if_none_match = request.META.get('HTTP_IF_NONE_MATCH') if if_none_match == etag: return HttpResponseNotModified() - f = None - if settings.CACHE_TILES: - dirname = os.path.sep.join((settings.TILES_ROOT, update_cache_key, level, str(zoom), str(x), str(y))) - filename = os.path.sep.join((dirname, access_cache_key+'.'+format)) + data = None + tile_dirname, last_update_filename, tile_filename, tile_cache_update_cache_key = '', '', '', '' - try: - f = open(filename, 'rb') - except FileNotFoundError: - pass + # get tile cache last update + if settings.CACHE_TILES: + tile_dirname = os.path.sep.join((settings.TILES_ROOT, str(level), str(zoom), str(x), str(y))) + last_update_filename = os.path.join(tile_dirname, 'last_update') + tile_filename = os.path.join(tile_dirname, renderer.access_cache_key+'.'+format) + + # get tile cache last update + tile_cache_update_cache_key = 'mapdata:tile-cache-update:%d-%d-%d-%d' % (level, zoom, x, y) + tile_cache_update = cache.get(tile_cache_update_cache_key, None) + if tile_cache_update is None: + try: + with open(last_update_filename) as f: + tile_cache_update = f.read() + except FileNotFoundError: + pass + + if tile_cache_update != update_cache_key: + os.system('rm -rf '+os.path.join(tile_dirname, '*')) + else: + try: + with open(tile_filename, 'rb') as f: + data = f.read() + except FileNotFoundError: + pass content_type = 'image/svg+xml' if format == 'svg' else 'image/png' - if not settings.CACHE_TILES or f is None: - try: - renderer.check_level() - except Level.DoesNotExist: - raise Http404 - + if data is None: svg = renderer.render() if format == 'svg': data = svg.get_xml() @@ -72,13 +95,12 @@ def tile(request, level, zoom, x, y, format): raise ValueError if settings.CACHE_TILES: - # noinspection PyUnboundLocalVariable - os.makedirs(dirname, exist_ok=True) - # noinspection PyUnboundLocalVariable - with open(filename, filemode) as f: + os.makedirs(tile_dirname, exist_ok=True) + with open(tile_filename, filemode) as f: f.write(data) - else: - data = f.read() + with open(last_update_filename, 'w') as f: + f.write(update_cache_key) + cache.get(tile_cache_update_cache_key, update_cache_key, 60) pr.disable() s = open('/tmp/profiled', 'w')