2025-summer/team-3
12
Fork 0
Code Issues Pull requests Projects Releases Activity

refactor tile code in meny places to allow reusing it

Browse source
This commit is contained in:
Laura Klünder 2017-11-21 00:15:49 +01:00
parent a503c0de6a
commit bf98f54b55
8 changed files with 162 additions and 161 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/c3nav/mapdata/management/commands/rendermap.py
View file

@ -68,9 +68,7 @@ class Command(BaseCommand):
full_levels=options['full_levels'])
filename = os.path.join(settings.RENDER_ROOT,
'level_%s_%s.%s' % (level.short_label,
renderer.access_cache_key.replace('_', '-'),
options['filetype']))
'level_%s.%s' % (level.short_label, options['filetype']))
render = renderer.render(get_engine(options['filetype']), center=not options['no_center'])
data = render.render(filename)

54
src/c3nav/mapdata/render/renderer.py
View file

@ -1,15 +1,13 @@
from itertools import chain
from django.core.cache import cache
from django.utils.functional import cached_property
from shapely import prepared
from shapely.geometry import box
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models import Level
from c3nav.mapdata.render.engines.base import FillAttribs, StrokeAttribs
from c3nav.mapdata.render.geometry import hybrid_union
from c3nav.mapdata.render.renderdata import LevelRenderData
from c3nav.mapdata.utils.cache import AccessRestrictionAffected, MapHistory
class MapRenderer:
@ -30,59 +28,25 @@ class MapRenderer:
def bbox(self):
return box(self.minx-1, self.miny-1, self.maxx+1, self.maxy+1)
@cached_property
def level_render_data(self):
return LevelRenderData.get(self.level)
@cached_property
def last_update(self):
return MapHistory.open_level_cached(self.level, 'composite').last_update(self.minx, self.miny,
self.maxx, self.maxy)
@cached_property
def update_cache_key(self):
return MapUpdate.build_cache_key(*self.last_update)
@cached_property
def affected_access_restrictions(self):
cache_key = 'mapdata:affected-ars-%.2f-%.2f-%.2f-%.2f:%s' % (self.minx, self.miny, self.maxx, self.maxy,
self.update_cache_key)
result = cache.get(cache_key, None)
if result is None:
result = set(AccessRestrictionAffected.open_level_cached(self.level, 'composite')[self.minx:self.maxx,
self.miny:self.maxy])
cache.set(cache_key, result, 120)
return result
@cached_property
def unlocked_access_restrictions(self):
return self.affected_access_restrictions & self.access_permissions
@cached_property
def access_cache_key(self):
return '_'.join(str(i) for i in sorted(self.unlocked_access_restrictions)) or '0'
@cached_property
def cache_key(self):
return self.update_cache_key + ':' + self.access_cache_key
def render(self, engine_cls, center=True):
engine = engine_cls(self.width, self.height, self.minx, self.miny,
scale=self.scale, buffer=1, background='#DCDCDC', center=center)
# add no access restriction to “unlocked“ access restrictions so lookup gets easier
unlocked_access_restrictions = self.unlocked_access_restrictions | set([None])
access_permissions = self.access_permissions | set([None])
bbox = prepared.prep(self.bbox)
level_render_data = LevelRenderData.get(self.level)
if self.full_levels:
levels = tuple(chain(*(
tuple(sublevel for sublevel in LevelRenderData.get(level.pk).levels
if sublevel.pk == level.pk or sublevel.on_top_of_id == level.pk)
for level in self.level_render_data.levels if level.on_top_of_id is None
for level in level_render_data.levels if level.on_top_of_id is None
)))
else:
levels = self.level_render_data.levels
levels = level_render_data.levels
min_altitude = min(chain(*(tuple(area.altitude for area in geoms.altitudeareas)
for geoms in levels)))
@ -97,10 +61,10 @@ class MapRenderer:
# hide indoor and outdoor rooms if their access restriction was not unlocked
add_walls = hybrid_union(tuple(area for access_restriction, area in geoms.restricted_spaces_indoors.items()
if access_restriction not in unlocked_access_restrictions))
if access_restriction not in access_permissions))
crop_areas = hybrid_union(
tuple(area for access_restriction, area in geoms.restricted_spaces_outdoors.items()
if access_restriction not in unlocked_access_restrictions)
if access_restriction not in access_permissions)
).union(add_walls)
if not_full_levels:
@ -129,7 +93,7 @@ class MapRenderer:
for color, areas in altitudearea.colors.items():
# only select ground colors if their access restriction is unlocked
areas = tuple(area for access_restriction, area in areas.items()
if access_restriction in unlocked_access_restrictions)
if access_restriction in access_permissions)
if areas:
i += 1
engine.add_geometry(hybrid_union(areas), fill=FillAttribs(color),

57
src/c3nav/mapdata/render/utils.py
View file

@ -1,57 +0,0 @@
import base64
import hashlib
import hmac
import time
from django.conf import settings
from django.core.cache import cache
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models.access import AccessPermission
def get_render_level_ids(cache_key=None):
if cache_key is None:
cache_key = MapUpdate.current_cache_key()
cache_key = 'mapdata:render-level-ids:'+cache_key
levels = cache.get(cache_key, None)
if levels is None:
levels = set(Level.objects.values_list('pk', flat=True))
cache.set(cache_key, levels, 300)
return levels
def set_tile_access_cookie(request, response):
access_permissions = AccessPermission.get_for_request(request)
if access_permissions:
value = '-'.join(str(i) for i in access_permissions)+':'+str(int(time.time())+60)
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, value+':'+signed, max_age=60)
else:
response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME)
def get_tile_access_cookie(request):
try:
cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME]
except KeyError:
return set()
try:
access_permissions, expire, signed = cookie.split(':')
except ValueError:
return set()
value = access_permissions+':'+expire
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
if signed != signed_verify:
return set()
if int(expire) < time.time():
return set()
return set(int(i) for i in access_permissions.split('-'))

7
src/c3nav/mapdata/urls.py
View file

@ -1,10 +1,11 @@
from django.conf.urls import url
from c3nav.mapdata.views import cache_package, history, tile, tile_access
from c3nav.mapdata.views import get_cache_package, map_history, tile, tile_access
urlpatterns = [
url(r'^(?P<level>\d+)/(?P<zoom>\d+)/(?P<x>-?\d+)/(?P<y>-?\d+).png$', tile, name='mapdata.tile'),
url(r'^history/(?P<level>\d+)/(?P<mode>base|composite).(?P<format>png|data)$', history, name='mapdata.history'),
url(r'^cache/package(?P<filetype>\.tar|\.tar\.gz|\.tar\.xz)$', cache_package, name='mapdata.cache_package'),
url(r'^history/(?P<level>\d+)/(?P<mode>base|composite)\.(?P<filetype>png|data)$', map_history,
name='mapdata.map_history'),
url(r'^cache/package\.(?P<filetype>tar|tar\.gz|tar\.xz)$', get_cache_package, name='mapdata.cache_package'),
url(r'^tile_access$', tile_access, name='mapdata.tile_access'),
]

31
src/c3nav/mapdata/utils/cache/package.py vendored
View file

@ -1,9 +1,12 @@
import os
import struct
import threading
from collections import namedtuple
from io import BytesIO
from tarfile import TarFile, TarInfo
from django.conf import settings
from c3nav.mapdata.utils.cache import AccessRestrictionAffected, GeometryIndexed, MapHistory
CachePackageLevel = namedtuple('CachePackageLevel', ('history', 'restrictions'))
@ -69,3 +72,31 @@ class CachePackage:
)
return cls(bounds, levels)
@classmethod
def open(cls, filename=None):
if filename is None:
filename = os.path.join(settings.CACHE_ROOT, 'package.tar')
return cls.read(open(filename, 'rb'))
cached = None
cache_key = None
cache_lock = threading.Lock()
@classmethod
def open_cached(cls):
with cls.cache_lock:
from c3nav.mapdata.models import MapUpdate
cache_key = MapUpdate.current_processed_cache_key()
if cls.cache_key != cache_key:
cls.cache_key = cache_key
cls.cached = None
if cls.cached is None:
cls.cached = cls.open()
return cls.cached
def bounds_valid(self, minx, miny, maxx, maxy):
return (minx <= self.bounds[2] and maxx >= self.bounds[0] and
miny <= self.bounds[3] and maxy >= self.bounds[1])

53
src/c3nav/mapdata/utils/tiles.py Normal file
View file

@ -0,0 +1,53 @@
import base64
import hashlib
import hmac
import time
def get_tile_bounds(zoom, x, y):
size = 256 / 2 ** zoom
minx = size * x
miny = size * (-y - 1)
maxx = minx + size
maxy = miny + size
# add one pixel so tiles can overlap to avoid rendering bugs in chrome or webkit
maxx += size / 256
miny -= size / 256
return minx, miny, maxx, maxy
def build_tile_access_cookie(access_permissions, tile_secret):
value = '-'.join(str(i) for i in access_permissions) + ':' + str(int(time.time()) + 60)
key = hashlib.sha1(tile_secret.encode()).digest()
signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
return value + ':' + signed
def parse_tile_access_cookie(cookie, tile_secret):
try:
access_permissions, expire, signed = cookie.split(':')
except ValueError:
return set()
value = access_permissions + ':' + expire
key = hashlib.sha1(tile_secret).digest()
signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
if signed != signed_verify:
return set()
if int(expire) < time.time():
return set()
return set(int(i) for i in access_permissions.split('-'))
def build_base_cache_key(last_update):
return '%x-%x' % last_update
def build_access_cache_key(access_permissions: set):
return '-'.join(str(i) for i in sorted(access_permissions)) or '0'
def build_tile_etag(level_id, zoom, x, y, base_cache_key, access_cache_key, tile_secret):
return '"' + base64.b64encode(hashlib.sha256(
('%d-%d-%d-%d:%s:%s:%s' % (level_id, zoom, x, y, base_cache_key, access_cache_key, tile_secret)).encode()
).digest()).decode() + '"'

110
src/c3nav/mapdata/views.py
View file

@ -1,24 +1,39 @@
import base64
import hashlib
import os
from itertools import chain
from functools import wraps
from wsgiref.util import FileWrapper
from django.conf import settings
from django.core.cache import cache
from django.core.exceptions import PermissionDenied
from django.core.signing import b64_encode
from django.http import Http404, HttpResponse, HttpResponseNotModified, StreamingHttpResponse
from django.shortcuts import get_object_or_404
from django.views.decorators.http import etag
from shapely.geometry import box
from c3nav.mapdata.middleware import no_language
from c3nav.mapdata.models import Level, MapUpdate, Source
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models.access import AccessPermission
from c3nav.mapdata.render.engines import ImageRenderEngine
from c3nav.mapdata.render.renderer import MapRenderer
from c3nav.mapdata.render.utils import get_render_level_ids, get_tile_access_cookie, set_tile_access_cookie
from c3nav.mapdata.utils.cache import MapHistory
from c3nav.mapdata.utils.cache import CachePackage, MapHistory
from c3nav.mapdata.utils.tiles import (build_access_cache_key, build_base_cache_key, build_tile_access_cookie,
build_tile_etag, get_tile_bounds, parse_tile_access_cookie)
def set_tile_access_cookie(func):
@wraps(func)
def wrapper(request, *args, **kwargs):
response = func(request, *args, **kwargs)
access_permissions = AccessPermission.get_for_request(request)
if access_permissions:
bla = build_tile_access_cookie(access_permissions, settings.SECRET_TILE_KEY)
response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, bla, max_age=60)
else:
response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME)
return response
return wrapper
@no_language()
@ -27,43 +42,41 @@ def tile(request, level, zoom, x, y):
if not (0 <= zoom <= 10):
raise Http404
# calculate bounds
x, y = int(x), int(y)
size = 256/2**zoom
minx = size * x
miny = size * (-y-1)
maxx = minx + size
maxy = miny + size
cache_package = CachePackage.open_cached()
# add one pixel so tiles can overlap to avoid rendering bugs in chrome or webkit
maxx += size / 256
miny -= size / 256
# error 404 if tiles is out of bounds
bounds = Source.max_bounds()
if not box(*chain(*bounds)).intersects(box(minx, miny, maxx, maxy)):
# check if bounds are valid
x = int(x)
y = int(y)
minx, miny, maxx, maxy = get_tile_bounds(zoom, x, y)
if not cache_package.bounds_valid(minx, miny, maxx, maxy):
raise Http404
# is this a valid level?
cache_key = MapUpdate.current_cache_key()
# get level
level = int(level)
if level not in get_render_level_ids(cache_key):
level_data = cache_package.levels.get(level)
if level_data is None:
raise Http404
# decode access permissions
access_permissions = get_tile_access_cookie(request)
try:
cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME]
except KeyError:
access_permissions = set()
else:
access_permissions = parse_tile_access_cookie(cookie, settings.SECRET_TILE_KEY)
# init renderer
renderer = MapRenderer(level, minx, miny, maxx, maxy, scale=2 ** zoom, access_permissions=access_permissions)
tile_cache_key = renderer.cache_key
update_cache_key = renderer.update_cache_key
# only access permissions that are affecting this tile
access_permissions &= set(level_data.restrictions[minx:miny, maxx:maxy])
# build cache keys
last_update = level_data.history.last_update(minx, miny, maxx, maxy)
base_cache_key = build_base_cache_key(last_update)
access_cache_key = build_access_cache_key(access_permissions)
# check browser cache
etag = '"'+b64_encode(hashlib.sha256(
('%d-%d-%d-%d:%s:%s' % (level, zoom, x, y, tile_cache_key, settings.SECRET_TILE_KEY)).encode()
).digest()).decode()+'"'
tile_etag = build_tile_etag(level, zoom, x, y, base_cache_key, access_cache_key, settings.SECRET_TILE_KEY)
if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
if if_none_match == etag:
if if_none_match == tile_etag:
return HttpResponseNotModified()
data = None
@ -71,12 +84,12 @@ def tile(request, level, zoom, x, y):
# get tile cache last update
if settings.CACHE_TILES:
tile_dirname = os.path.sep.join((settings.TILES_ROOT, str(level), str(zoom), str(x), str(y)))
tile_dirname = os.path.sep.join((settings.TILES_ROOT, str(level_data), str(zoom), str(x), str(y)))
last_update_filename = os.path.join(tile_dirname, 'last_update')
tile_filename = os.path.join(tile_dirname, renderer.access_cache_key+'.png')
tile_filename = os.path.join(tile_dirname, access_cache_key+'.png')
# get tile cache last update
tile_cache_update_cache_key = 'mapdata:tile-cache-update:%d-%d-%d-%d' % (level, zoom, x, y)
tile_cache_update_cache_key = 'mapdata:tile-cache-update:%d-%d-%d-%d' % (level_data, zoom, x, y)
tile_cache_update = cache.get(tile_cache_update_cache_key, None)
if tile_cache_update is None:
try:
@ -85,7 +98,7 @@ def tile(request, level, zoom, x, y):
except FileNotFoundError:
pass
if tile_cache_update != update_cache_key:
if tile_cache_update != base_cache_key:
os.system('rm -rf '+os.path.join(tile_dirname, '*'))
else:
try:
@ -95,6 +108,7 @@ def tile(request, level, zoom, x, y):
pass
if data is None:
renderer = MapRenderer(level, minx, miny, maxx, maxy, scale=2 ** zoom, access_permissions=access_permissions)
image = renderer.render(ImageRenderEngine)
data = image.render()
@ -103,30 +117,28 @@ def tile(request, level, zoom, x, y):
with open(tile_filename, 'wb') as f:
f.write(data)
with open(last_update_filename, 'w') as f:
f.write(update_cache_key)
cache.get(tile_cache_update_cache_key, update_cache_key, 60)
f.write(base_cache_key)
cache.get(tile_cache_update_cache_key, base_cache_key, 60)
response = HttpResponse(data, 'image/png')
response['ETag'] = etag
response['X-ETag-Unencoded'] = '%d-%d-%d-%d:%s' % (level, zoom, x, y, tile_cache_key)
response['ETag'] = tile_etag
response['Cache-Control'] = 'no-cache'
response['Vary'] = 'Cookie'
response['X-Access-Restrictions'] = ', '.join(str(s) for s in renderer.unlocked_access_restrictions) or '0'
return response
@no_language()
@set_tile_access_cookie
def tile_access(request):
response = HttpResponse(content_type='text/plain')
set_tile_access_cookie(request, response)
response['Cache-Control'] = 'no-cache'
return response
@etag(lambda *args, **kwargs: MapUpdate.current_processed_cache_key())
@no_language()
def history(request, level, mode, format):
def map_history(request, level, mode, filetype):
if not request.user.is_superuser:
raise PermissionDenied
level = get_object_or_404(Level, pk=level)
@ -135,10 +147,10 @@ def history(request, level, mode, format):
raise Http404
history = MapHistory.open_level(level.pk, mode)
if format == 'png':
if filetype == 'png':
response = HttpResponse(content_type='image/png')
history.to_image().save(response, format='PNG')
elif format == 'data':
elif filetype == 'data':
response = HttpResponse(content_type='application/octet-stream')
history.write(response)
else:
@ -152,7 +164,7 @@ encoded_tile_secret = base64.b64encode(settings.SECRET_TILE_KEY.encode()).decode
@etag(lambda *args, **kwargs: MapUpdate.current_processed_cache_key())
@no_language()
def cache_package(request, filetype):
def get_cache_package(request, filetype):
x_tile_secret = request.META.get('HTTP_X_TILE_SECRET')
if x_tile_secret:
if x_tile_secret != encoded_tile_secret:
@ -160,14 +172,14 @@ def cache_package(request, filetype):
elif not request.user.is_superuser:
raise PermissionDenied
filename = os.path.join(settings.CACHE_ROOT, 'package'+filetype)
filename = os.path.join(settings.CACHE_ROOT, 'package.'+filetype)
f = open(filename, 'rb')
f.seek(0, os.SEEK_END)
size = f.tell()
f.seek(0)
content_type = 'application/' + {'.tar': 'x-tar', '.tar.gz': 'gzip', '.tar.xz': 'x-xz'}[filetype]
content_type = 'application/' + {'tar': 'x-tar', 'tar.gz': 'gzip', 'tar.xz': 'x-xz'}[filetype]
response = StreamingHttpResponse(FileWrapper(f), content_type=content_type)
response['Content-Length'] = size

7
src/c3nav/site/views.py
View file

@ -15,8 +15,8 @@ from c3nav.mapdata.models import Location, Source
from c3nav.mapdata.models.access import AccessPermission
from c3nav.mapdata.models.level import Level
from c3nav.mapdata.models.locations import LocationRedirect, SpecificLocation
from c3nav.mapdata.render.utils import set_tile_access_cookie
from c3nav.mapdata.utils.locations import get_location_by_slug_for_request
from c3nav.mapdata.views import set_tile_access_cookie
ctype_mapping = {
'yes': ('up', 'down'),
@ -99,6 +99,7 @@ def get_levels(request) -> Mapping[int, Level]:
return levels
@set_tile_access_cookie
def map_index(request, mode=None, slug=None, slug2=None, level=None, x=None, y=None, zoom=None):
origin = None
destination = None
@ -145,9 +146,7 @@ def map_index(request, mode=None, slug=None, slug2=None, level=None, x=None, y=N
'levels': json.dumps(tuple(levels.values()), separators=(',', ':')),
'state': json.dumps(state, separators=(',', ':')),
}
response = render(request, 'site/map.html', ctx)
set_tile_access_cookie(request, response)
return response
return render(request, 'site/map.html', ctx)
def main(request, location=None, origin=None, destination=None):