diff --git a/src/c3nav/mapdata/management/commands/rendermap.py b/src/c3nav/mapdata/management/commands/rendermap.py index f320eccb..fb9f18ac 100644 --- a/src/c3nav/mapdata/management/commands/rendermap.py +++ b/src/c3nav/mapdata/management/commands/rendermap.py @@ -68,9 +68,7 @@ class Command(BaseCommand): full_levels=options['full_levels']) filename = os.path.join(settings.RENDER_ROOT, - 'level_%s_%s.%s' % (level.short_label, - renderer.access_cache_key.replace('_', '-'), - options['filetype'])) + 'level_%s.%s' % (level.short_label, options['filetype'])) render = renderer.render(get_engine(options['filetype']), center=not options['no_center']) data = render.render(filename) diff --git a/src/c3nav/mapdata/render/renderer.py b/src/c3nav/mapdata/render/renderer.py index e7267697..da8c0c42 100644 --- a/src/c3nav/mapdata/render/renderer.py +++ b/src/c3nav/mapdata/render/renderer.py @@ -1,15 +1,13 @@ from itertools import chain -from django.core.cache import cache from django.utils.functional import cached_property from shapely import prepared from shapely.geometry import box -from c3nav.mapdata.models import Level, MapUpdate +from c3nav.mapdata.models import Level from c3nav.mapdata.render.engines.base import FillAttribs, StrokeAttribs from c3nav.mapdata.render.geometry import hybrid_union from c3nav.mapdata.render.renderdata import LevelRenderData -from c3nav.mapdata.utils.cache import AccessRestrictionAffected, MapHistory class MapRenderer: @@ -30,59 +28,25 @@ class MapRenderer: def bbox(self): return box(self.minx-1, self.miny-1, self.maxx+1, self.maxy+1) - @cached_property - def level_render_data(self): - return LevelRenderData.get(self.level) - - @cached_property - def last_update(self): - return MapHistory.open_level_cached(self.level, 'composite').last_update(self.minx, self.miny, - self.maxx, self.maxy) - - @cached_property - def update_cache_key(self): - return MapUpdate.build_cache_key(*self.last_update) - - @cached_property - def affected_access_restrictions(self): - cache_key = 'mapdata:affected-ars-%.2f-%.2f-%.2f-%.2f:%s' % (self.minx, self.miny, self.maxx, self.maxy, - self.update_cache_key) - result = cache.get(cache_key, None) - if result is None: - result = set(AccessRestrictionAffected.open_level_cached(self.level, 'composite')[self.minx:self.maxx, - self.miny:self.maxy]) - cache.set(cache_key, result, 120) - return result - - @cached_property - def unlocked_access_restrictions(self): - return self.affected_access_restrictions & self.access_permissions - - @cached_property - def access_cache_key(self): - return '_'.join(str(i) for i in sorted(self.unlocked_access_restrictions)) or '0' - - @cached_property - def cache_key(self): - return self.update_cache_key + ':' + self.access_cache_key - def render(self, engine_cls, center=True): engine = engine_cls(self.width, self.height, self.minx, self.miny, scale=self.scale, buffer=1, background='#DCDCDC', center=center) # add no access restriction to “unlocked“ access restrictions so lookup gets easier - unlocked_access_restrictions = self.unlocked_access_restrictions | set([None]) + access_permissions = self.access_permissions | set([None]) bbox = prepared.prep(self.bbox) + level_render_data = LevelRenderData.get(self.level) + if self.full_levels: levels = tuple(chain(*( tuple(sublevel for sublevel in LevelRenderData.get(level.pk).levels if sublevel.pk == level.pk or sublevel.on_top_of_id == level.pk) - for level in self.level_render_data.levels if level.on_top_of_id is None + for level in level_render_data.levels if level.on_top_of_id is None ))) else: - levels = self.level_render_data.levels + levels = level_render_data.levels min_altitude = min(chain(*(tuple(area.altitude for area in geoms.altitudeareas) for geoms in levels))) @@ -97,10 +61,10 @@ class MapRenderer: # hide indoor and outdoor rooms if their access restriction was not unlocked add_walls = hybrid_union(tuple(area for access_restriction, area in geoms.restricted_spaces_indoors.items() - if access_restriction not in unlocked_access_restrictions)) + if access_restriction not in access_permissions)) crop_areas = hybrid_union( tuple(area for access_restriction, area in geoms.restricted_spaces_outdoors.items() - if access_restriction not in unlocked_access_restrictions) + if access_restriction not in access_permissions) ).union(add_walls) if not_full_levels: @@ -129,7 +93,7 @@ class MapRenderer: for color, areas in altitudearea.colors.items(): # only select ground colors if their access restriction is unlocked areas = tuple(area for access_restriction, area in areas.items() - if access_restriction in unlocked_access_restrictions) + if access_restriction in access_permissions) if areas: i += 1 engine.add_geometry(hybrid_union(areas), fill=FillAttribs(color), diff --git a/src/c3nav/mapdata/render/utils.py b/src/c3nav/mapdata/render/utils.py deleted file mode 100644 index 9bc4046f..00000000 --- a/src/c3nav/mapdata/render/utils.py +++ /dev/null @@ -1,57 +0,0 @@ -import base64 -import hashlib -import hmac -import time - -from django.conf import settings -from django.core.cache import cache - -from c3nav.mapdata.models import Level, MapUpdate -from c3nav.mapdata.models.access import AccessPermission - - -def get_render_level_ids(cache_key=None): - if cache_key is None: - cache_key = MapUpdate.current_cache_key() - cache_key = 'mapdata:render-level-ids:'+cache_key - levels = cache.get(cache_key, None) - if levels is None: - levels = set(Level.objects.values_list('pk', flat=True)) - cache.set(cache_key, levels, 300) - return levels - - -def set_tile_access_cookie(request, response): - access_permissions = AccessPermission.get_for_request(request) - - if access_permissions: - value = '-'.join(str(i) for i in access_permissions)+':'+str(int(time.time())+60) - key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest() - signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode() - response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, value+':'+signed, max_age=60) - else: - response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME) - - -def get_tile_access_cookie(request): - try: - cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME] - except KeyError: - return set() - - try: - access_permissions, expire, signed = cookie.split(':') - except ValueError: - return set() - - value = access_permissions+':'+expire - - key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest() - signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode() - if signed != signed_verify: - return set() - - if int(expire) < time.time(): - return set() - - return set(int(i) for i in access_permissions.split('-')) diff --git a/src/c3nav/mapdata/urls.py b/src/c3nav/mapdata/urls.py index 2b01be76..214c104e 100644 --- a/src/c3nav/mapdata/urls.py +++ b/src/c3nav/mapdata/urls.py @@ -1,10 +1,11 @@ from django.conf.urls import url -from c3nav.mapdata.views import cache_package, history, tile, tile_access +from c3nav.mapdata.views import get_cache_package, map_history, tile, tile_access urlpatterns = [ url(r'^(?P\d+)/(?P\d+)/(?P-?\d+)/(?P-?\d+).png$', tile, name='mapdata.tile'), - url(r'^history/(?P\d+)/(?Pbase|composite).(?Ppng|data)$', history, name='mapdata.history'), - url(r'^cache/package(?P\.tar|\.tar\.gz|\.tar\.xz)$', cache_package, name='mapdata.cache_package'), + url(r'^history/(?P\d+)/(?Pbase|composite)\.(?Ppng|data)$', map_history, + name='mapdata.map_history'), + url(r'^cache/package\.(?Ptar|tar\.gz|tar\.xz)$', get_cache_package, name='mapdata.cache_package'), url(r'^tile_access$', tile_access, name='mapdata.tile_access'), ] diff --git a/src/c3nav/mapdata/utils/cache/package.py b/src/c3nav/mapdata/utils/cache/package.py index e5f823bf..514a4595 100644 --- a/src/c3nav/mapdata/utils/cache/package.py +++ b/src/c3nav/mapdata/utils/cache/package.py @@ -1,9 +1,12 @@ import os import struct +import threading from collections import namedtuple from io import BytesIO from tarfile import TarFile, TarInfo +from django.conf import settings + from c3nav.mapdata.utils.cache import AccessRestrictionAffected, GeometryIndexed, MapHistory CachePackageLevel = namedtuple('CachePackageLevel', ('history', 'restrictions')) @@ -69,3 +72,31 @@ class CachePackage: ) return cls(bounds, levels) + + @classmethod + def open(cls, filename=None): + if filename is None: + filename = os.path.join(settings.CACHE_ROOT, 'package.tar') + return cls.read(open(filename, 'rb')) + + cached = None + cache_key = None + cache_lock = threading.Lock() + + @classmethod + def open_cached(cls): + with cls.cache_lock: + from c3nav.mapdata.models import MapUpdate + cache_key = MapUpdate.current_processed_cache_key() + if cls.cache_key != cache_key: + cls.cache_key = cache_key + cls.cached = None + + if cls.cached is None: + cls.cached = cls.open() + + return cls.cached + + def bounds_valid(self, minx, miny, maxx, maxy): + return (minx <= self.bounds[2] and maxx >= self.bounds[0] and + miny <= self.bounds[3] and maxy >= self.bounds[1]) diff --git a/src/c3nav/mapdata/utils/tiles.py b/src/c3nav/mapdata/utils/tiles.py new file mode 100644 index 00000000..b36ab038 --- /dev/null +++ b/src/c3nav/mapdata/utils/tiles.py @@ -0,0 +1,53 @@ +import base64 +import hashlib +import hmac +import time + + +def get_tile_bounds(zoom, x, y): + size = 256 / 2 ** zoom + minx = size * x + miny = size * (-y - 1) + maxx = minx + size + maxy = miny + size + + # add one pixel so tiles can overlap to avoid rendering bugs in chrome or webkit + maxx += size / 256 + miny -= size / 256 + return minx, miny, maxx, maxy + + +def build_tile_access_cookie(access_permissions, tile_secret): + value = '-'.join(str(i) for i in access_permissions) + ':' + str(int(time.time()) + 60) + key = hashlib.sha1(tile_secret.encode()).digest() + signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode() + return value + ':' + signed + + +def parse_tile_access_cookie(cookie, tile_secret): + try: + access_permissions, expire, signed = cookie.split(':') + except ValueError: + return set() + value = access_permissions + ':' + expire + key = hashlib.sha1(tile_secret).digest() + signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode() + if signed != signed_verify: + return set() + if int(expire) < time.time(): + return set() + return set(int(i) for i in access_permissions.split('-')) + + +def build_base_cache_key(last_update): + return '%x-%x' % last_update + + +def build_access_cache_key(access_permissions: set): + return '-'.join(str(i) for i in sorted(access_permissions)) or '0' + + +def build_tile_etag(level_id, zoom, x, y, base_cache_key, access_cache_key, tile_secret): + return '"' + base64.b64encode(hashlib.sha256( + ('%d-%d-%d-%d:%s:%s:%s' % (level_id, zoom, x, y, base_cache_key, access_cache_key, tile_secret)).encode() + ).digest()).decode() + '"' diff --git a/src/c3nav/mapdata/views.py b/src/c3nav/mapdata/views.py index 139087e3..d37cfda9 100644 --- a/src/c3nav/mapdata/views.py +++ b/src/c3nav/mapdata/views.py @@ -1,24 +1,39 @@ import base64 -import hashlib import os -from itertools import chain +from functools import wraps from wsgiref.util import FileWrapper from django.conf import settings from django.core.cache import cache from django.core.exceptions import PermissionDenied -from django.core.signing import b64_encode from django.http import Http404, HttpResponse, HttpResponseNotModified, StreamingHttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.http import etag -from shapely.geometry import box from c3nav.mapdata.middleware import no_language -from c3nav.mapdata.models import Level, MapUpdate, Source +from c3nav.mapdata.models import Level, MapUpdate +from c3nav.mapdata.models.access import AccessPermission from c3nav.mapdata.render.engines import ImageRenderEngine from c3nav.mapdata.render.renderer import MapRenderer -from c3nav.mapdata.render.utils import get_render_level_ids, get_tile_access_cookie, set_tile_access_cookie -from c3nav.mapdata.utils.cache import MapHistory +from c3nav.mapdata.utils.cache import CachePackage, MapHistory +from c3nav.mapdata.utils.tiles import (build_access_cache_key, build_base_cache_key, build_tile_access_cookie, + build_tile_etag, get_tile_bounds, parse_tile_access_cookie) + + +def set_tile_access_cookie(func): + @wraps(func) + def wrapper(request, *args, **kwargs): + response = func(request, *args, **kwargs) + + access_permissions = AccessPermission.get_for_request(request) + if access_permissions: + bla = build_tile_access_cookie(access_permissions, settings.SECRET_TILE_KEY) + response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, bla, max_age=60) + else: + response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME) + + return response + return wrapper @no_language() @@ -27,43 +42,41 @@ def tile(request, level, zoom, x, y): if not (0 <= zoom <= 10): raise Http404 - # calculate bounds - x, y = int(x), int(y) - size = 256/2**zoom - minx = size * x - miny = size * (-y-1) - maxx = minx + size - maxy = miny + size + cache_package = CachePackage.open_cached() - # add one pixel so tiles can overlap to avoid rendering bugs in chrome or webkit - maxx += size / 256 - miny -= size / 256 - - # error 404 if tiles is out of bounds - bounds = Source.max_bounds() - if not box(*chain(*bounds)).intersects(box(minx, miny, maxx, maxy)): + # check if bounds are valid + x = int(x) + y = int(y) + minx, miny, maxx, maxy = get_tile_bounds(zoom, x, y) + if not cache_package.bounds_valid(minx, miny, maxx, maxy): raise Http404 - # is this a valid level? - cache_key = MapUpdate.current_cache_key() + # get level level = int(level) - if level not in get_render_level_ids(cache_key): + level_data = cache_package.levels.get(level) + if level_data is None: raise Http404 # decode access permissions - access_permissions = get_tile_access_cookie(request) + try: + cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME] + except KeyError: + access_permissions = set() + else: + access_permissions = parse_tile_access_cookie(cookie, settings.SECRET_TILE_KEY) - # init renderer - renderer = MapRenderer(level, minx, miny, maxx, maxy, scale=2 ** zoom, access_permissions=access_permissions) - tile_cache_key = renderer.cache_key - update_cache_key = renderer.update_cache_key + # only access permissions that are affecting this tile + access_permissions &= set(level_data.restrictions[minx:miny, maxx:maxy]) + + # build cache keys + last_update = level_data.history.last_update(minx, miny, maxx, maxy) + base_cache_key = build_base_cache_key(last_update) + access_cache_key = build_access_cache_key(access_permissions) # check browser cache - etag = '"'+b64_encode(hashlib.sha256( - ('%d-%d-%d-%d:%s:%s' % (level, zoom, x, y, tile_cache_key, settings.SECRET_TILE_KEY)).encode() - ).digest()).decode()+'"' + tile_etag = build_tile_etag(level, zoom, x, y, base_cache_key, access_cache_key, settings.SECRET_TILE_KEY) if_none_match = request.META.get('HTTP_IF_NONE_MATCH') - if if_none_match == etag: + if if_none_match == tile_etag: return HttpResponseNotModified() data = None @@ -71,12 +84,12 @@ def tile(request, level, zoom, x, y): # get tile cache last update if settings.CACHE_TILES: - tile_dirname = os.path.sep.join((settings.TILES_ROOT, str(level), str(zoom), str(x), str(y))) + tile_dirname = os.path.sep.join((settings.TILES_ROOT, str(level_data), str(zoom), str(x), str(y))) last_update_filename = os.path.join(tile_dirname, 'last_update') - tile_filename = os.path.join(tile_dirname, renderer.access_cache_key+'.png') + tile_filename = os.path.join(tile_dirname, access_cache_key+'.png') # get tile cache last update - tile_cache_update_cache_key = 'mapdata:tile-cache-update:%d-%d-%d-%d' % (level, zoom, x, y) + tile_cache_update_cache_key = 'mapdata:tile-cache-update:%d-%d-%d-%d' % (level_data, zoom, x, y) tile_cache_update = cache.get(tile_cache_update_cache_key, None) if tile_cache_update is None: try: @@ -85,7 +98,7 @@ def tile(request, level, zoom, x, y): except FileNotFoundError: pass - if tile_cache_update != update_cache_key: + if tile_cache_update != base_cache_key: os.system('rm -rf '+os.path.join(tile_dirname, '*')) else: try: @@ -95,6 +108,7 @@ def tile(request, level, zoom, x, y): pass if data is None: + renderer = MapRenderer(level, minx, miny, maxx, maxy, scale=2 ** zoom, access_permissions=access_permissions) image = renderer.render(ImageRenderEngine) data = image.render() @@ -103,30 +117,28 @@ def tile(request, level, zoom, x, y): with open(tile_filename, 'wb') as f: f.write(data) with open(last_update_filename, 'w') as f: - f.write(update_cache_key) - cache.get(tile_cache_update_cache_key, update_cache_key, 60) + f.write(base_cache_key) + cache.get(tile_cache_update_cache_key, base_cache_key, 60) response = HttpResponse(data, 'image/png') - response['ETag'] = etag - response['X-ETag-Unencoded'] = '%d-%d-%d-%d:%s' % (level, zoom, x, y, tile_cache_key) + response['ETag'] = tile_etag response['Cache-Control'] = 'no-cache' response['Vary'] = 'Cookie' - response['X-Access-Restrictions'] = ', '.join(str(s) for s in renderer.unlocked_access_restrictions) or '0' return response @no_language() +@set_tile_access_cookie def tile_access(request): response = HttpResponse(content_type='text/plain') - set_tile_access_cookie(request, response) response['Cache-Control'] = 'no-cache' return response @etag(lambda *args, **kwargs: MapUpdate.current_processed_cache_key()) @no_language() -def history(request, level, mode, format): +def map_history(request, level, mode, filetype): if not request.user.is_superuser: raise PermissionDenied level = get_object_or_404(Level, pk=level) @@ -135,10 +147,10 @@ def history(request, level, mode, format): raise Http404 history = MapHistory.open_level(level.pk, mode) - if format == 'png': + if filetype == 'png': response = HttpResponse(content_type='image/png') history.to_image().save(response, format='PNG') - elif format == 'data': + elif filetype == 'data': response = HttpResponse(content_type='application/octet-stream') history.write(response) else: @@ -152,7 +164,7 @@ encoded_tile_secret = base64.b64encode(settings.SECRET_TILE_KEY.encode()).decode @etag(lambda *args, **kwargs: MapUpdate.current_processed_cache_key()) @no_language() -def cache_package(request, filetype): +def get_cache_package(request, filetype): x_tile_secret = request.META.get('HTTP_X_TILE_SECRET') if x_tile_secret: if x_tile_secret != encoded_tile_secret: @@ -160,14 +172,14 @@ def cache_package(request, filetype): elif not request.user.is_superuser: raise PermissionDenied - filename = os.path.join(settings.CACHE_ROOT, 'package'+filetype) + filename = os.path.join(settings.CACHE_ROOT, 'package.'+filetype) f = open(filename, 'rb') f.seek(0, os.SEEK_END) size = f.tell() f.seek(0) - content_type = 'application/' + {'.tar': 'x-tar', '.tar.gz': 'gzip', '.tar.xz': 'x-xz'}[filetype] + content_type = 'application/' + {'tar': 'x-tar', 'tar.gz': 'gzip', 'tar.xz': 'x-xz'}[filetype] response = StreamingHttpResponse(FileWrapper(f), content_type=content_type) response['Content-Length'] = size diff --git a/src/c3nav/site/views.py b/src/c3nav/site/views.py index 8a54702d..02b17194 100644 --- a/src/c3nav/site/views.py +++ b/src/c3nav/site/views.py @@ -15,8 +15,8 @@ from c3nav.mapdata.models import Location, Source from c3nav.mapdata.models.access import AccessPermission from c3nav.mapdata.models.level import Level from c3nav.mapdata.models.locations import LocationRedirect, SpecificLocation -from c3nav.mapdata.render.utils import set_tile_access_cookie from c3nav.mapdata.utils.locations import get_location_by_slug_for_request +from c3nav.mapdata.views import set_tile_access_cookie ctype_mapping = { 'yes': ('up', 'down'), @@ -99,6 +99,7 @@ def get_levels(request) -> Mapping[int, Level]: return levels +@set_tile_access_cookie def map_index(request, mode=None, slug=None, slug2=None, level=None, x=None, y=None, zoom=None): origin = None destination = None @@ -145,9 +146,7 @@ def map_index(request, mode=None, slug=None, slug2=None, level=None, x=None, y=N 'levels': json.dumps(tuple(levels.values()), separators=(',', ':')), 'state': json.dumps(state, separators=(',', ':')), } - response = render(request, 'site/map.html', ctx) - set_tile_access_cookie(request, response) - return response + return render(request, 'site/map.html', ctx) def main(request, location=None, origin=None, destination=None):