cleanup render code to make room for the opengl rendering engine

This commit is contained in:
Laura Klünder 2017-11-04 23:21:36 +01:00
parent f7291ff237
commit df451f9143
12 changed files with 207 additions and 159 deletions

View file

@ -1 +0,0 @@
from c3nav.mapdata.render.svg import SVGRenderer # noqa

View file

@ -0,0 +1,3 @@
from c3nav.mapdata.render.image.renderer import ImageRenderer # noqa
from c3nav.mapdata.render.image.utils import (get_render_level_ids, set_tile_access_cookie, # noqa
get_tile_access_cookie) # noqa

View file

@ -1,64 +1,11 @@
import base64
import hashlib
import hmac
import pickle
import time
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from shapely.ops import unary_union
from c3nav.mapdata.cache import MapHistory
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models.access import AccessPermission
def get_render_level_ids(cache_key=None):
if cache_key is None:
cache_key = MapUpdate.current_cache_key()
cache_key = 'mapdata:render-level-ids:'+cache_key
levels = cache.get(cache_key, None)
if levels is None:
levels = set(Level.objects.values_list('pk', flat=True))
cache.set(cache_key, levels, 300)
return levels
def set_tile_access_cookie(request, response):
access_permissions = AccessPermission.get_for_request(request)
if access_permissions:
value = '-'.join(str(i) for i in access_permissions)+':'+str(int(time.time())+60)
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, value+':'+signed, max_age=60)
else:
response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME)
def get_tile_access_cookie(request):
try:
cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME]
except KeyError:
return set()
try:
access_permissions, expire, signed = cookie.split(':')
except ValueError:
return set()
value = access_permissions+':'+expire
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
if signed != signed_verify:
return set()
if int(expire) < time.time():
return set()
return set(int(i) for i in access_permissions.split('-'))
class AltitudeAreaGeometries:

View file

@ -0,0 +1,74 @@
import math
from abc import ABC, abstractmethod
from typing import Optional
class FillAttribs:
__slots__ = ('color', 'opacity')
def __init__(self, color, opacity=None):
self.color = color
self.opacity = opacity
class StrokeAttribs:
__slots__ = ('color', 'width', 'min_px', 'opacity')
def __init__(self, color, width, min_px=None, opacity=None):
self.color = color
self.width = width
self.min_px = min_px
self.opacity = opacity
class RenderEngine(ABC):
# draw an svg image. supports pseudo-3D shadow-rendering
def __init__(self, width: int, height: int, xoff=0, yoff=0, scale=1, buffer=0, background='#FFFFFF'):
self.width = width
self.height = height
self.minx = xoff
self.miny = yoff
self.scale = scale
self.buffer = int(math.ceil(buffer*self.scale))
self.background = background
self.maxx = self.minx + width / scale
self.maxy = self.miny + height / scale
# how many pixels around the image should be added and later cropped (otherwise rsvg does not blur correctly)
self.buffer = int(math.ceil(buffer*self.scale))
self.buffered_width = self.width + 2 * self.buffer
self.buffered_height = self.height + 2 * self.buffer
self.background_rgb = tuple(int(background[i:i + 2], 16) for i in range(1, 6, 2))
@abstractmethod
def get_png(self) -> bytes:
# render the image to png.
pass
def add_geometry(self, geometry, fill: Optional[FillAttribs] = None, stroke: Optional[StrokeAttribs] = None,
filter=None, clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
# draw a shapely geometry with a given style
# if altitude is set, the geometry will get a calculated shadow relative to the other geometries
# if elevation is set, the geometry will get a shadow with exactly this elevation
# if fill_color is set, filter out geometries that cannot be filled
if fill is not None:
try:
geometry.geoms
except AttributeError:
if not hasattr(geometry, 'exterior'):
return
else:
geometry = type(geometry)(tuple(geom for geom in geometry.geoms if hasattr(geom, 'exterior')))
if geometry.is_empty:
return
self._add_geometry(geometry=geometry, fill=fill, stroke=stroke, filter=filter, clip_path=clip_path,
altitude=altitude, elevation=elevation, shape_cache_key=shape_cache_key)
@abstractmethod
def _add_geometry(self, geometry, fill: Optional[FillAttribs] = None, stroke: Optional[StrokeAttribs] = None,
filter=None, clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
pass

View file

@ -0,0 +1,267 @@
import io
import re
import subprocess
import zlib
from itertools import chain
from typing import Optional
import numpy as np
from django.conf import settings
from django.core import checks
from PIL import Image
from shapely.affinity import translate
from shapely.geometry import LineString, Polygon
from shapely.ops import unary_union
# import gobject-inspect, cairo and rsvg if the native rsvg SVG_RENDERER should be used
from c3nav.mapdata.render.image.engines.base import FillAttribs, RenderEngine, StrokeAttribs
if settings.SVG_RENDERER == 'rsvg':
import pgi
import cairocffi
pgi.require_version('Rsvg', '2.0')
from pgi.repository import Rsvg
@checks.register()
def check_svg_renderer(app_configs, **kwargs):
errors = []
if settings.SVG_RENDERER not in ('rsvg', 'rsvg-convert', 'inkscape'):
errors.append(
checks.Error(
'Invalid SVG renderer: '+settings.SVG_RENDERER,
obj='settings.SVG_RENDERER',
id='c3nav.mapdata.E001',
)
)
return errors
class SVGEngine(RenderEngine):
# draw an svg image. supports pseudo-3D shadow-rendering
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# create base elements and counter for clip path ids
self.g = ''
self.defs = ''
self.clip_path_i = 0
# keep track which area of the image has which altitude currently
self.altitudes = {}
self.last_altitude = None
# for fast numpy operations
self.np_scale = np.array((self.scale, -self.scale))
self.np_offset = np.array((-self.minx * self.scale, self.maxy * self.scale))
# keep track of created blur filters to avoid duplicates
self.blurs = set()
self._create_geometry_cache = {}
def get_xml(self, buffer=False):
# get the root <svg> element as an ElementTree element, with or without buffer
if buffer:
width_px = self._trim_decimals(str(self.buffered_width))
height_px = self._trim_decimals(str(self.buffered_height))
offset_px = self._trim_decimals(str(-self.buffer))
attribs = ' viewBox="' + ' '.join((offset_px, offset_px, width_px, height_px)) + '"' if buffer else ''
else:
width_px = self._trim_decimals(str(self.width))
height_px = self._trim_decimals(str(self.height))
attribs = ''
result = ('<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" '
'width="'+width_px+'" height="'+height_px+'"'+attribs+'>')
if self.defs:
result += '<defs>'+self.defs+'</defs>'
if self.g:
result += '<g>'+self.g+'</g>'
result += '</svg>'
return result
def get_png(self):
# render the image to png. returns bytes if f is None, otherwise it calls f.write()
if self.width == 256 and self.height == 256 and not self.g:
# create empty tile png with minimal size, indexed color palette with only one entry
plte = b'PLTE' + bytearray(self.background_rgb)
return (b'\x89PNG\r\n\x1a\n' +
b'\x00\x00\x00\rIHDR\x00\x00\x01\x00\x00\x00\x01\x00\x01\x03\x00\x00\x00f\xbc:%\x00\x00\x00\x03' +
plte + zlib.crc32(plte).to_bytes(4, byteorder='big') +
b'\x00\x00\x00\x1fIDATh\xde\xed\xc1\x01\r\x00\x00\x00\xc2\xa0\xf7Om\x0e7\xa0\x00\x00\x00\x00\x00' +
b'\x00\x00\x00\xbe\r!\x00\x00\x01\x7f\x19\x9c\xa7\x00\x00\x00\x00IEND\xaeB`\x82')
if settings.SVG_RENDERER == 'rsvg':
# create buffered surfaces
buffered_surface = cairocffi.SVGSurface(None, self.buffered_width, self.buffered_height)
buffered_context = cairocffi.Context(buffered_surface)
# draw svg with rsvg
handle = Rsvg.Handle()
svg = handle.new_from_data(self.get_xml(buffer=True).encode())
svg.render_cairo(buffered_context)
# create cropped image
surface = buffered_surface.create_similar(cairocffi.CONTENT_COLOR, self.width, self.height)
context = cairocffi.Context(surface)
# set background color
context.set_source(cairocffi.SolidPattern(*(i/255 for i in self.background_rgb)))
context.paint()
# paste buffered immage with offset
context.set_source_surface(buffered_surface, -self.buffer, -self.buffer)
context.paint()
return surface.write_to_png()
elif settings.SVG_RENDERER == 'rsvg-convert':
p = subprocess.run(('rsvg-convert', '-b', self.background, '--format', 'png'),
input=self.get_xml(buffer=True).encode(), stdout=subprocess.PIPE, check=True)
png = io.BytesIO(p.stdout)
img = Image.open(png)
img = img.crop((self.buffer, self.buffer,
self.buffer + self.width,
self.buffer + self.height))
f = io.BytesIO()
img.save(f, 'PNG')
f.seek(0)
return f.read()
elif settings.SVG_RENDERER == 'inkscape':
p = subprocess.run(('inkscape', '-z', '-b', self.background, '-e', '/dev/stderr', '/dev/stdin'),
input=self.get_xml().encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
check=True)
png = p.stderr[p.stderr.index(b'\x89PNG'):]
return png
def _trim_decimals(self, data):
# remove trailing zeros from a decimal yes this is slow, but it greatly speeds up cairo rendering
return re.sub(r'([0-9]+)((\.[1-9])[0-9]+|\.[0-9]+)?', r'\1\2', data)
def _geometry_to_svg(self, geom):
# scale and move geometry and create svg code for it
if isinstance(geom, Polygon):
return ('<path d="' +
' '.join((('M %.1f %.1f L'+(' %.1f %.1f'*(len(ring.coords)-1))+' z') %
tuple((np.array(ring)*self.np_scale+self.np_offset).flatten()))
for ring in chain((geom.exterior,), geom.interiors))
+ '"/>').replace('.0 ', ' ')
if isinstance(geom, LineString):
return (('<path d="M %.1f %.1f L'+(' %.1f %.1f'*(len(geom.coords)-1))+'"/>') %
tuple((np.array(geom)*self.np_scale+self.np_offset).flatten())).replace('.0 ', ' ')
try:
geoms = geom.geoms
except AttributeError:
return ''
return ''.join(self._geometry_to_svg(g) for g in geoms)
def _create_geometry(self, geometry, attribs='', tag='g', cache_key=None):
# convert a shapely geometry into an svg xml element
result = None
if cache_key is not None:
result = self._create_geometry_cache.get(cache_key, None)
if result is None:
result = self._geometry_to_svg(geometry)
if cache_key is not None:
self._create_geometry_cache[cache_key] = result
return '<'+tag+attribs+'>'+result+'</'+tag+'>'
def register_clip_path(self, geometry):
defid = 'clip'+str(self.clip_path_i)
self.defs += self._create_geometry(geometry, ' id="'+defid+'"', tag='clipPath')
self.clip_path_i += 1
return defid
def add_shadow(self, geometry, elevation, clip_path=None):
# add a shadow for the given geometry with the given elevation and, optionally, a clip path
elevation = float(min(elevation, 2))
blur_radius = elevation / 3 * 0.25
shadow_geom = translate(geometry.buffer(blur_radius),
xoff=(elevation / 3 * 0.12), yoff=-(elevation / 3 * 0.12))
if clip_path is not None:
if shadow_geom.distance(clip_path) >= blur_radius:
return
blur_id = 'blur'+str(int(elevation*100))
if elevation not in self.blurs:
self.defs += ('<filter id="'+blur_id+'" width="200%" height="200%" x="-50%" y="-50%">'
'<feGaussianBlur stdDeviation="'+str(blur_radius * self.scale)+'"/>'
'</filter>')
self.blurs.add(elevation)
attribs = ' filter="url(#'+blur_id+')" fill="#000" fill-opacity="0.2"'
if clip_path:
attribs += ' clip-path="url(#'+self.register_clip_path(clip_path)+')"'
shadow = self._create_geometry(shadow_geom, attribs)
self.g += shadow
def clip_altitudes(self, new_geometry, new_altitude=None):
# registrer new geometry with specific (or no) altitude
# a geometry with no altitude will reset the altitude information of its area as if nothing was ever there
for altitude, geometry in tuple(self.altitudes.items()):
if altitude != new_altitude:
self.altitudes[altitude] = geometry.difference(new_geometry)
if self.altitudes[altitude].is_empty:
self.altitudes.pop(altitude)
if new_altitude is not None:
if self.last_altitude is not None and self.last_altitude > new_altitude:
raise ValueError('Altitudes have to be ascending.')
self.last_altitude = new_altitude
if new_altitude in self.altitudes:
self.altitudes[new_altitude] = unary_union([self.altitudes[new_altitude], new_geometry])
else:
self.altitudes[new_altitude] = new_geometry
def _add_geometry(self, geometry, fill: Optional[FillAttribs] = None, stroke: Optional[StrokeAttribs] = None,
filter=None, clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
if fill:
attribs = ' fill="'+(fill.color)+'"'
if fill.opacity:
attribs += ' fill-opacity="'+str(fill.opacity)[:4]+'"'
else:
attribs = ' fill="none"'
if stroke:
width = stroke.width*self.scale
if stroke.min_px:
width = max(width, stroke.min_px)
attribs += ' stroke-width="' + self._trim_decimals(str(width)) + '" stroke="' + stroke.color + '"'
if stroke.opacity:
attribs += ' stroke-opacity="'+str(stroke.opacity)[:4]+'"'
if filter:
attribs += ' filter="url(#'+filter+')"'
if clip_path:
attribs += ' clip-path="url(#'+clip_path+')"'
if geometry is not None:
if False:
# old shadow rendering. currently needs too much resources
if altitude is not None or elevation is not None:
if elevation is not None:
if elevation:
self.add_shadow(geometry, elevation)
else:
for other_altitude, other_geom in self.altitudes.items():
self.add_shadow(geometry, altitude-other_altitude, clip_path=other_geom)
self.clip_altitudes(geometry, altitude)
else:
if elevation is not None:
self.add_shadow(geometry, elevation)
element = self._create_geometry(geometry, attribs, cache_key=shape_cache_key)
else:
element = '<rect width="100%" height="100%"'+attribs+'>'
self.g += element
return element

View file

@ -6,11 +6,12 @@ from shapely.ops import unary_union
from c3nav.mapdata.cache import MapHistory
from c3nav.mapdata.models import MapUpdate
from c3nav.mapdata.render.base import get_level_render_data
from c3nav.mapdata.utils.svg import SVGImage
from c3nav.mapdata.render.image.data import get_level_render_data
from c3nav.mapdata.render.image.engines.base import FillAttribs, StrokeAttribs
from c3nav.mapdata.render.image.engines.svg import SVGEngine
class SVGRenderer:
class ImageRenderer:
def __init__(self, level, minx, miny, maxx, maxy, scale=1, access_permissions=None):
self.level = level
self.minx = minx
@ -20,6 +21,9 @@ class SVGRenderer:
self.scale = scale
self.access_permissions = access_permissions
self.width = int(round((maxx - minx) * scale))
self.height = int(round((maxy - miny) * scale))
@cached_property
def bbox(self):
return box(self.minx-1, self.miny-1, self.maxx+1, self.maxy+1)
@ -61,8 +65,8 @@ class SVGRenderer:
return self.update_cache_key + ':' + self.access_cache_key
def render(self):
svg = SVGImage(bounds=((self.minx, self.miny), (self.maxx, self.maxy)),
scale=self.scale, buffer=1, background_color='#DCDCDC')
svg = SVGEngine(self.width, self.height, self.minx, self.miny,
scale=self.scale, buffer=1, background='#DCDCDC')
# add no access restriction to “unlocked“ access restrictions so lookup gets easier
unlocked_access_restrictions = self.unlocked_access_restrictions | set([None])
@ -86,15 +90,15 @@ class SVGRenderer:
# shadows are directly calculated and added by the SVGImage class
for altitudearea in geoms.altitudeareas:
svg.add_geometry(bbox.intersection(altitudearea.geometry.difference(crop_areas)),
fill_color='#eeeeee', altitude=altitudearea.altitude,
stroke_width=0.05, stroke_px=0.2, stroke_color='rgba(0, 0, 0, 0.15)')
altitude=altitudearea.altitude, fill=FillAttribs('#eeeeee'),
stroke=StrokeAttribs('rgba(0, 0, 0, 0.15)', 0.05, min_px=0.2))
for color, areas in altitudearea.colors.items():
# only select ground colors if their access restriction is unlocked
areas = tuple(area for access_restriction, area in areas.items()
if access_restriction in unlocked_access_restrictions)
if areas:
svg.add_geometry(bbox.intersection(unary_union(areas)), fill_color=color)
svg.add_geometry(bbox.intersection(unary_union(areas)), fill=FillAttribs(color))
# add walls, stroke_px makes sure that all walls are at least 1px thick on all zoom levels,
walls = None
@ -102,13 +106,13 @@ class SVGRenderer:
walls = bbox.intersection(geoms.walls.union(add_walls))
if walls is not None:
svg.add_geometry(walls, elevation=default_height, fill_color='#aaaaaa')
svg.add_geometry(walls, elevation=default_height, fill=FillAttribs('#aaaaaa'))
if not geoms.doors.is_empty:
svg.add_geometry(bbox.intersection(geoms.doors.difference(add_walls)),
fill_color='#ffffff', stroke_width=0.05, stroke_px=0.2, stroke_color='#ffffff')
svg.add_geometry(bbox.intersection(geoms.doors.difference(add_walls)), fill=FillAttribs('#ffffff'),
stroke=StrokeAttribs('#ffffff', 0.05, min_px=0.2))
if walls is not None:
svg.add_geometry(walls, stroke_width=0.05, stroke_px=0.2, stroke_color='#666666')
svg.add_geometry(walls, stroke=StrokeAttribs('#666666', 0.05, min_px=0.2))
return svg

View file

@ -0,0 +1,57 @@
import base64
import hashlib
import hmac
import time
from django.conf import settings
from django.core.cache import cache
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models.access import AccessPermission
def get_render_level_ids(cache_key=None):
if cache_key is None:
cache_key = MapUpdate.current_cache_key()
cache_key = 'mapdata:render-level-ids:'+cache_key
levels = cache.get(cache_key, None)
if levels is None:
levels = set(Level.objects.values_list('pk', flat=True))
cache.set(cache_key, levels, 300)
return levels
def set_tile_access_cookie(request, response):
access_permissions = AccessPermission.get_for_request(request)
if access_permissions:
value = '-'.join(str(i) for i in access_permissions)+':'+str(int(time.time())+60)
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, value+':'+signed, max_age=60)
else:
response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME)
def get_tile_access_cookie(request):
try:
cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME]
except KeyError:
return set()
try:
access_permissions, expire, signed = cookie.split(':')
except ValueError:
return set()
value = access_permissions+':'+expire
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
if signed != signed_verify:
return set()
if int(expire) < time.time():
return set()
return set(int(i) for i in access_permissions.split('-'))