tile view: use new intelligent caching
This commit is contained in:
Laura Klünder
parent
7640e77c14
commit
6b6e3afa7b
5 changed files with 110 additions and 32 deletions
|
|
@ -5,6 +5,7 @@ from itertools import chain
|
|||
|
||||
import numpy as np
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db.models.signals import m2m_changed, post_delete
|
||||
from PIL import Image
|
||||
from shapely import prepared
|
||||
|
|
@ -64,6 +65,17 @@ class MapHistory:
|
|||
def open_level(cls, level_id, mode, default_update=None):
|
||||
return cls.open(cls.level_filename(level_id, mode), default_update)
|
||||
|
||||
@classmethod
|
||||
def open_level_cached(cls, level_id, mode, cache_key=None):
|
||||
if cache_key is None:
|
||||
cache_key = MapUpdate.current_cache_key()
|
||||
cache_key = 'mapdata:map-history-%d-%s:%s' % (level_id, mode, cache_key)
|
||||
result = cache.get(cache_key, None)
|
||||
if result is None:
|
||||
result = cls.open_level(level_id, mode)
|
||||
cache.set(cache_key, result, 120)
|
||||
return result
|
||||
|
||||
def save(self, filename=None):
|
||||
if filename is None:
|
||||
filename = self.filename
|
||||
|
|
@ -205,6 +217,17 @@ class MapHistory:
|
|||
|
||||
return Image.fromarray(np.flip(image_data, axis=0), 'L')
|
||||
|
||||
def last_update(self, minx, miny, maxx, maxy):
|
||||
res = self.resolution
|
||||
height, width = self.data.shape
|
||||
minx = max(int(math.floor(minx/res)), self.x)-self.x
|
||||
miny = max(int(math.floor(miny/res)), self.y)-self.y
|
||||
maxx = min(int(math.ceil(maxx/res)), self.x+width)-self.x
|
||||
maxy = min(int(math.ceil(maxy/res)), self.y+height)-self.y
|
||||
if minx >= maxx or miny >= maxy:
|
||||
return self.updates[0]
|
||||
return self.updates[self.data[miny:maxy, minx:maxx].max()]
|
||||
|
||||
|
||||
class GeometryChangeTracker:
|
||||
def __init__(self):
|
||||
|
|
|
|||
|
|
@ -41,11 +41,14 @@ class MapUpdate(models.Model):
|
|||
|
||||
@property
|
||||
def cache_key(self):
|
||||
return int_to_base36(self.pk)+'_'+int_to_base36(int(make_naive(self.datetime).timestamp()))
|
||||
return self.build_cache_key(self.pk, int(make_naive(self.datetime).timestamp()))
|
||||
|
||||
@classmethod
|
||||
def current_cache_key(cls):
|
||||
pk, timestamp = cls.last_update()
|
||||
return cls.build_cache_key(*cls.last_update())
|
||||
|
||||
@staticmethod
|
||||
def build_cache_key(pk, timestamp):
|
||||
return int_to_base36(pk)+'_'+int_to_base36(timestamp)
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -8,6 +8,17 @@ from c3nav.mapdata.cache import MapHistory
|
|||
from c3nav.mapdata.models import Level, MapUpdate
|
||||
|
||||
|
||||
def get_render_level_ids(cache_key=None):
|
||||
if cache_key is None:
|
||||
cache_key = MapUpdate.current_cache_key()
|
||||
cache_key = 'mapdata:render-level-ids:'+cache_key
|
||||
levels = cache.get(cache_key, None)
|
||||
if levels is None:
|
||||
levels = set(Level.objects.values_list('pk', flat=True))
|
||||
cache.set(cache_key, levels, 300)
|
||||
return levels
|
||||
|
||||
|
||||
class AltitudeAreaGeometries:
|
||||
def __init__(self, altitudearea=None, colors=None):
|
||||
if altitudearea is not None:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
from django.core.cache import cache
|
||||
from django.utils.functional import cached_property
|
||||
from shapely import prepared
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import unary_union
|
||||
|
||||
from c3nav.mapdata.cache import MapHistory
|
||||
from c3nav.mapdata.models import MapUpdate
|
||||
from c3nav.mapdata.render.base import get_level_render_data
|
||||
from c3nav.mapdata.utils.svg import SVGImage
|
||||
|
||||
|
|
@ -25,13 +28,25 @@ class SVGRenderer:
|
|||
def level_render_data(self):
|
||||
return get_level_render_data(self.level)
|
||||
|
||||
def check_level(self):
|
||||
return self.level_render_data
|
||||
@cached_property
|
||||
def last_update(self):
|
||||
return MapHistory.open_level_cached(self.level, 'render').last_update(self.minx, self.miny,
|
||||
self.maxx, self.maxy)
|
||||
|
||||
@cached_property
|
||||
def update_cache_key(self):
|
||||
return MapUpdate.build_cache_key(*self.last_update)
|
||||
|
||||
@cached_property
|
||||
def affected_access_restrictions(self):
|
||||
return set(ar for ar, area in self.level_render_data.access_restriction_affected.items()
|
||||
if area.intersects(self.bbox))
|
||||
cache_key = 'mapdata:affected-ars-%.2f-%.2f-%.2f-%.2f:%s' % (self.minx, self.miny, self.maxx, self.maxy,
|
||||
self.update_cache_key)
|
||||
result = cache.get(cache_key, None)
|
||||
if result is None:
|
||||
result = set(ar for ar, area in self.level_render_data.access_restriction_affected.items()
|
||||
if area.intersects(self.bbox))
|
||||
cache.set(cache_key, result, 120)
|
||||
return result
|
||||
|
||||
@cached_property
|
||||
def unlocked_access_restrictions(self):
|
||||
|
|
@ -43,6 +58,10 @@ class SVGRenderer:
|
|||
def access_cache_key(self):
|
||||
return '_'.join(str(i) for i in sorted(self.unlocked_access_restrictions)) or '0'
|
||||
|
||||
@cached_property
|
||||
def cache_key(self):
|
||||
return self.update_cache_key + ':' + self.access_cache_key
|
||||
|
||||
def render(self):
|
||||
svg = SVGImage(bounds=((self.miny, self.minx), (self.maxy, self.maxx)), scale=self.scale, buffer=1)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.core.exceptions import PermissionDenied
|
||||
from django.http import Http404, HttpResponse, HttpResponseNotModified
|
||||
from django.shortcuts import get_object_or_404
|
||||
|
|
@ -8,6 +9,7 @@ from shapely.geometry import box
|
|||
|
||||
from c3nav.mapdata.cache import MapHistory
|
||||
from c3nav.mapdata.models import Level, MapUpdate, Source
|
||||
from c3nav.mapdata.render.base import get_render_level_ids
|
||||
from c3nav.mapdata.render.svg import SVGRenderer
|
||||
|
||||
|
||||
|
|
@ -21,8 +23,7 @@ def tile(request, level, zoom, x, y, format):
|
|||
if not (0 <= zoom <= 10):
|
||||
raise Http404
|
||||
|
||||
bounds = Source.max_bounds()
|
||||
|
||||
# calculate bounds
|
||||
x, y = int(x), int(y)
|
||||
size = 256/2**zoom
|
||||
minx = size * x
|
||||
|
|
@ -30,37 +31,59 @@ def tile(request, level, zoom, x, y, format):
|
|||
maxx = minx + size
|
||||
maxy = miny + size
|
||||
|
||||
# error 404 if tiles is out of bounds
|
||||
bounds = Source.max_bounds()
|
||||
if not box(bounds[0][1], bounds[0][0], bounds[1][1], bounds[1][0]).intersects(box(minx, miny, maxx, maxy)):
|
||||
raise Http404
|
||||
|
||||
renderer = SVGRenderer(level, miny, minx, maxy, maxx, scale=2**zoom, user=request.user)
|
||||
# is this a valid level?
|
||||
cache_key = MapUpdate.current_cache_key()
|
||||
level = int(level)
|
||||
if level not in get_render_level_ids(cache_key):
|
||||
raise Http404
|
||||
|
||||
update_cache_key = MapUpdate.current_cache_key()
|
||||
access_cache_key = renderer.access_cache_key
|
||||
etag = update_cache_key+'_'+access_cache_key
|
||||
# init renderer
|
||||
renderer = SVGRenderer(level, miny, minx, maxy, maxx, scale=2 ** zoom, user=request.user)
|
||||
tile_cache_key = renderer.cache_key
|
||||
update_cache_key = renderer.update_cache_key
|
||||
|
||||
# check browser cache
|
||||
etag = tile_cache_key
|
||||
if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
|
||||
if if_none_match == etag:
|
||||
return HttpResponseNotModified()
|
||||
|
||||
f = None
|
||||
if settings.CACHE_TILES:
|
||||
dirname = os.path.sep.join((settings.TILES_ROOT, update_cache_key, level, str(zoom), str(x), str(y)))
|
||||
filename = os.path.sep.join((dirname, access_cache_key+'.'+format))
|
||||
data = None
|
||||
tile_dirname, last_update_filename, tile_filename, tile_cache_update_cache_key = '', '', '', ''
|
||||
|
||||
try:
|
||||
f = open(filename, 'rb')
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
# get tile cache last update
|
||||
if settings.CACHE_TILES:
|
||||
tile_dirname = os.path.sep.join((settings.TILES_ROOT, str(level), str(zoom), str(x), str(y)))
|
||||
last_update_filename = os.path.join(tile_dirname, 'last_update')
|
||||
tile_filename = os.path.join(tile_dirname, renderer.access_cache_key+'.'+format)
|
||||
|
||||
# get tile cache last update
|
||||
tile_cache_update_cache_key = 'mapdata:tile-cache-update:%d-%d-%d-%d' % (level, zoom, x, y)
|
||||
tile_cache_update = cache.get(tile_cache_update_cache_key, None)
|
||||
if tile_cache_update is None:
|
||||
try:
|
||||
with open(last_update_filename) as f:
|
||||
tile_cache_update = f.read()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
if tile_cache_update != update_cache_key:
|
||||
os.system('rm -rf '+os.path.join(tile_dirname, '*'))
|
||||
else:
|
||||
try:
|
||||
with open(tile_filename, 'rb') as f:
|
||||
data = f.read()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
content_type = 'image/svg+xml' if format == 'svg' else 'image/png'
|
||||
|
||||
if not settings.CACHE_TILES or f is None:
|
||||
try:
|
||||
renderer.check_level()
|
||||
except Level.DoesNotExist:
|
||||
raise Http404
|
||||
|
||||
if data is None:
|
||||
svg = renderer.render()
|
||||
if format == 'svg':
|
||||
data = svg.get_xml()
|
||||
|
|
@ -72,13 +95,12 @@ def tile(request, level, zoom, x, y, format):
|
|||
raise ValueError
|
||||
|
||||
if settings.CACHE_TILES:
|
||||
# noinspection PyUnboundLocalVariable
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
# noinspection PyUnboundLocalVariable
|
||||
with open(filename, filemode) as f:
|
||||
os.makedirs(tile_dirname, exist_ok=True)
|
||||
with open(tile_filename, filemode) as f:
|
||||
f.write(data)
|
||||
else:
|
||||
data = f.read()
|
||||
with open(last_update_filename, 'w') as f:
|
||||
f.write(update_cache_key)
|
||||
cache.get(tile_cache_update_cache_key, update_cache_key, 60)
|
||||
|
||||
pr.disable()
|
||||
s = open('/tmp/profiled', 'w')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue