cleanup render code to make room for the opengl rendering engine

This commit is contained in:
Laura Klünder 2017-11-04 23:21:36 +01:00
parent f7291ff237
commit df451f9143
12 changed files with 207 additions and 159 deletions

View file

@ -15,7 +15,6 @@ from shapely.ops import cascaded_union
from c3nav.mapdata.models.locations import SpecificLocation
from c3nav.mapdata.utils.geometry import assert_multipolygon
from c3nav.mapdata.utils.scad import add_indent, polygon_scad
from c3nav.mapdata.utils.svg import SVGImage
class LevelManager(models.Manager):
@ -125,6 +124,7 @@ class Level(SpecificLocation, models.Model):
svg.add_geometry(obstacle_geometries, fill_color='#999999')
def render_svg(self, request, effects=True, draw_spaces=None):
from c3nav.mapdata.render.image.engines.svg import SVGImage
from c3nav.mapdata.models import Source, Area, Door, Space
bounds = Source.max_bounds()

View file

@ -69,7 +69,7 @@ class MapUpdate(models.Model):
from c3nav.mapdata.cache import changed_geometries
changed_geometries.save(last_map_update, self.to_tuple)
from c3nav.mapdata.render.base import LevelRenderData
from c3nav.mapdata.render.image.data import LevelRenderData
LevelRenderData.rebuild()
cache.set('mapdata:last_update', self.to_tuple, 900)

View file

@ -1 +0,0 @@
from c3nav.mapdata.render.svg import SVGRenderer # noqa

View file

@ -0,0 +1,3 @@
from c3nav.mapdata.render.image.renderer import ImageRenderer # noqa
from c3nav.mapdata.render.image.utils import (get_render_level_ids, set_tile_access_cookie, # noqa
get_tile_access_cookie) # noqa

View file

@ -1,64 +1,11 @@
import base64
import hashlib
import hmac
import pickle
import time
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from shapely.ops import unary_union
from c3nav.mapdata.cache import MapHistory
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models.access import AccessPermission
def get_render_level_ids(cache_key=None):
if cache_key is None:
cache_key = MapUpdate.current_cache_key()
cache_key = 'mapdata:render-level-ids:'+cache_key
levels = cache.get(cache_key, None)
if levels is None:
levels = set(Level.objects.values_list('pk', flat=True))
cache.set(cache_key, levels, 300)
return levels
def set_tile_access_cookie(request, response):
access_permissions = AccessPermission.get_for_request(request)
if access_permissions:
value = '-'.join(str(i) for i in access_permissions)+':'+str(int(time.time())+60)
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, value+':'+signed, max_age=60)
else:
response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME)
def get_tile_access_cookie(request):
try:
cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME]
except KeyError:
return set()
try:
access_permissions, expire, signed = cookie.split(':')
except ValueError:
return set()
value = access_permissions+':'+expire
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
if signed != signed_verify:
return set()
if int(expire) < time.time():
return set()
return set(int(i) for i in access_permissions.split('-'))
class AltitudeAreaGeometries:

View file

@ -0,0 +1,74 @@
import math
from abc import ABC, abstractmethod
from typing import Optional
class FillAttribs:
__slots__ = ('color', 'opacity')
def __init__(self, color, opacity=None):
self.color = color
self.opacity = opacity
class StrokeAttribs:
__slots__ = ('color', 'width', 'min_px', 'opacity')
def __init__(self, color, width, min_px=None, opacity=None):
self.color = color
self.width = width
self.min_px = min_px
self.opacity = opacity
class RenderEngine(ABC):
# draw an svg image. supports pseudo-3D shadow-rendering
def __init__(self, width: int, height: int, xoff=0, yoff=0, scale=1, buffer=0, background='#FFFFFF'):
self.width = width
self.height = height
self.minx = xoff
self.miny = yoff
self.scale = scale
self.buffer = int(math.ceil(buffer*self.scale))
self.background = background
self.maxx = self.minx + width / scale
self.maxy = self.miny + height / scale
# how many pixels around the image should be added and later cropped (otherwise rsvg does not blur correctly)
self.buffer = int(math.ceil(buffer*self.scale))
self.buffered_width = self.width + 2 * self.buffer
self.buffered_height = self.height + 2 * self.buffer
self.background_rgb = tuple(int(background[i:i + 2], 16) for i in range(1, 6, 2))
@abstractmethod
def get_png(self) -> bytes:
# render the image to png.
pass
def add_geometry(self, geometry, fill: Optional[FillAttribs] = None, stroke: Optional[StrokeAttribs] = None,
filter=None, clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
# draw a shapely geometry with a given style
# if altitude is set, the geometry will get a calculated shadow relative to the other geometries
# if elevation is set, the geometry will get a shadow with exactly this elevation
# if fill_color is set, filter out geometries that cannot be filled
if fill is not None:
try:
geometry.geoms
except AttributeError:
if not hasattr(geometry, 'exterior'):
return
else:
geometry = type(geometry)(tuple(geom for geom in geometry.geoms if hasattr(geom, 'exterior')))
if geometry.is_empty:
return
self._add_geometry(geometry=geometry, fill=fill, stroke=stroke, filter=filter, clip_path=clip_path,
altitude=altitude, elevation=elevation, shape_cache_key=shape_cache_key)
@abstractmethod
def _add_geometry(self, geometry, fill: Optional[FillAttribs] = None, stroke: Optional[StrokeAttribs] = None,
filter=None, clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
pass

View file

@ -1,9 +1,9 @@
import io
import math
import re
import subprocess
import zlib
from itertools import chain
from typing import Optional
import numpy as np
from django.conf import settings
@ -14,6 +14,8 @@ from shapely.geometry import LineString, Polygon
from shapely.ops import unary_union
# import gobject-inspect, cairo and rsvg if the native rsvg SVG_RENDERER should be used
from c3nav.mapdata.render.image.engines.base import FillAttribs, RenderEngine, StrokeAttribs
if settings.SVG_RENDERER == 'rsvg':
import pgi
import cairocffi
@ -35,18 +37,10 @@ def check_svg_renderer(app_configs, **kwargs):
return errors
class SVGImage:
class SVGEngine(RenderEngine):
# draw an svg image. supports pseudo-3D shadow-rendering
def __init__(self, bounds, scale: float=1, buffer=0, background_color='#FFFFFF'):
# get image dimensions.
# note that these values describe the „viewport“ of the image, not its dimensions in pixels.
(self.left, self.bottom), (self.right, self.top) = bounds
self.width = self.right-self.left
self.height = self.top-self.bottom
self.scale = scale
# how many pixels around the image should be added and later cropped (otherwise rsvg does not blur correctly)
self.buffer_px = int(math.ceil(buffer*self.scale))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# create base elements and counter for clip path ids
self.g = ''
@ -59,28 +53,24 @@ class SVGImage:
# for fast numpy operations
self.np_scale = np.array((self.scale, -self.scale))
self.np_offset = np.array((-self.left*self.scale, self.top*self.scale))
self.np_offset = np.array((-self.minx * self.scale, self.maxy * self.scale))
# keep track of created blur filters to avoid duplicates
self.blurs = set()
self.background_color = background_color
self.background_color_rgb = tuple(int(background_color[i:i + 2], 16) for i in range(1, 6, 2))
self._create_geometry_cache = {}
def get_dimensions_px(self, buffer):
# get dimensions of the image in pixels, with or without buffer
width_px = self.width * self.scale + (self.buffer_px * 2 if buffer else 0)
height_px = self.height * self.scale + (self.buffer_px * 2 if buffer else 0)
return height_px, width_px
def get_xml(self, buffer=False):
# get the root <svg> element as an ElementTree element, with or without buffer
height_px, width_px = (self._trim_decimals(str(i)) for i in self.get_dimensions_px(buffer))
offset_px = self._trim_decimals(str(-self.buffer_px)) if buffer else '0'
attribs = ' viewBox="'+' '.join((offset_px, offset_px, width_px, height_px))+'"' if buffer else ''
if buffer:
width_px = self._trim_decimals(str(self.buffered_width))
height_px = self._trim_decimals(str(self.buffered_height))
offset_px = self._trim_decimals(str(-self.buffer))
attribs = ' viewBox="' + ' '.join((offset_px, offset_px, width_px, height_px)) + '"' if buffer else ''
else:
width_px = self._trim_decimals(str(self.width))
height_px = self._trim_decimals(str(self.height))
attribs = ''
result = ('<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" '
'width="'+width_px+'" height="'+height_px+'"'+attribs+'>')
@ -91,12 +81,12 @@ class SVGImage:
result += '</svg>'
return result
def get_png(self, f=None):
def get_png(self):
# render the image to png. returns bytes if f is None, otherwise it calls f.write()
if self.get_dimensions_px(buffer=False) == (256, 256) and not self.g:
if self.width == 256 and self.height == 256 and not self.g:
# create empty tile png with minimal size, indexed color palette with only one entry
plte = b'PLTE' + bytearray(self.background_color_rgb)
plte = b'PLTE' + bytearray(self.background_rgb)
return (b'\x89PNG\r\n\x1a\n' +
b'\x00\x00\x00\rIHDR\x00\x00\x01\x00\x00\x00\x01\x00\x01\x03\x00\x00\x00f\xbc:%\x00\x00\x00\x03' +
plte + zlib.crc32(plte).to_bytes(4, byteorder='big') +
@ -105,7 +95,7 @@ class SVGImage:
if settings.SVG_RENDERER == 'rsvg':
# create buffered surfaces
buffered_surface = cairocffi.SVGSurface(None, *(int(i) for i in self.get_dimensions_px(buffer=True)))
buffered_surface = cairocffi.SVGSurface(None, self.buffered_width, self.buffered_height)
buffered_context = cairocffi.Context(buffered_surface)
# draw svg with rsvg
@ -114,44 +104,39 @@ class SVGImage:
svg.render_cairo(buffered_context)
# create cropped image
surface = buffered_surface.create_similar(cairocffi.CONTENT_COLOR,
*(int(i) for i in self.get_dimensions_px(buffer=False)))
surface = buffered_surface.create_similar(cairocffi.CONTENT_COLOR, self.width, self.height)
context = cairocffi.Context(surface)
# set background color
context.set_source(cairocffi.SolidPattern(*(i/255 for i in self.background_color_rgb)))
context.set_source(cairocffi.SolidPattern(*(i/255 for i in self.background_rgb)))
context.paint()
# paste buffered immage with offset
context.set_source_surface(buffered_surface, -self.buffer_px, -self.buffer_px)
context.set_source_surface(buffered_surface, -self.buffer, -self.buffer)
context.paint()
if f is None:
return surface.write_to_png()
f.write(surface.write_to_png())
return surface.write_to_png()
elif settings.SVG_RENDERER == 'rsvg-convert':
p = subprocess.run(('rsvg-convert', '-b', self.background_color, '--format', 'png'),
p = subprocess.run(('rsvg-convert', '-b', self.background, '--format', 'png'),
input=self.get_xml(buffer=True).encode(), stdout=subprocess.PIPE, check=True)
png = io.BytesIO(p.stdout)
img = Image.open(png)
img = img.crop((self.buffer_px, self.buffer_px,
self.buffer_px + int(self.width * self.scale),
self.buffer_px + int(self.height * self.scale)))
if f is None:
f = io.BytesIO()
img.save(f, 'PNG')
f.seek(0)
return f.read()
img = img.crop((self.buffer, self.buffer,
self.buffer + self.width,
self.buffer + self.height))
f = io.BytesIO()
img.save(f, 'PNG')
f.seek(0)
return f.read()
elif settings.SVG_RENDERER == 'inkscape':
p = subprocess.run(('inkscape', '-z', '-b', self.background_color, '-e', '/dev/stderr', '/dev/stdin'),
p = subprocess.run(('inkscape', '-z', '-b', self.background, '-e', '/dev/stderr', '/dev/stdin'),
input=self.get_xml().encode(), stdout=subprocess.PIPE, stderr=subprocess.PIPE,
check=True)
png = p.stderr[p.stderr.index(b'\x89PNG'):]
if f is None:
return png
f.write(png)
return png
def _trim_decimals(self, data):
# remove trailing zeros from a decimal yes this is slow, but it greatly speeds up cairo rendering
@ -233,51 +218,30 @@ class SVGImage:
else:
self.altitudes[new_altitude] = new_geometry
def add_geometry(self, geometry=None, fill_color=None, fill_opacity=None, opacity=None, filter=None,
stroke_px=0.0, stroke_width=0.0, stroke_color=None, stroke_opacity=None, stroke_linejoin=None,
clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
# draw a shapely geometry with a given style
# if altitude is set, the geometry will get a calculated shadow relative to the other geometries
# if elevation is set, the geometry will get a shadow with exactly this elevation
def _add_geometry(self, geometry, fill: Optional[FillAttribs] = None, stroke: Optional[StrokeAttribs] = None,
filter=None, clip_path=None, altitude=None, elevation=None, shape_cache_key=None):
# if fill_color is set, filter out geometries that cannot be filled
if fill_color is not None:
try:
geometry.geoms
except AttributeError:
if not hasattr(geometry, 'exterior'):
return
else:
geometry = type(geometry)(tuple(geom for geom in geometry.geoms if hasattr(geom, 'exterior')))
if geometry.is_empty:
pass
if fill:
attribs = ' fill="'+(fill.color)+'"'
if fill.opacity:
attribs += ' fill-opacity="'+str(fill.opacity)[:4]+'"'
else:
attribs = ' fill="none"'
if stroke:
width = stroke.width*self.scale
if stroke.min_px:
width = max(width, stroke.min_px)
attribs += ' stroke-width="' + self._trim_decimals(str(width)) + '" stroke="' + stroke.color + '"'
if stroke.opacity:
attribs += ' stroke-opacity="'+str(stroke.opacity)[:4]+'"'
attribs = ' fill="'+(fill_color or 'none')+'"'
if fill_opacity:
attribs += ' fill-opacity="'+str(fill_opacity)[:4]+'"'
if stroke_width:
width = stroke_width*self.scale
if stroke_px:
width = max(width, stroke_px)
attribs += ' stroke-width="' + self._trim_decimals(str(width)) + '"'
elif stroke_px:
attribs += ' stroke-width="'+self._trim_decimals(str(stroke_px))+'"'
if stroke_color:
attribs += ' stroke="'+stroke_color+'"'
if stroke_opacity:
attribs += ' stroke-opacity="'+str(stroke_opacity)[:4]+'"'
if stroke_linejoin:
attribs += ' stroke-linejoin="'+stroke_linejoin+'"'
if opacity:
attribs += ' opacity="'+str(opacity)[:4]+'"'
if filter:
attribs += ' filter="url(#'+filter+')"'
if clip_path:
attribs += ' clip-path="url(#'+clip_path+')"'
if geometry is not None:
if not geometry:
return
if False:
# old shadow rendering. currently needs too much resources

View file

@ -6,11 +6,12 @@ from shapely.ops import unary_union
from c3nav.mapdata.cache import MapHistory
from c3nav.mapdata.models import MapUpdate
from c3nav.mapdata.render.base import get_level_render_data
from c3nav.mapdata.utils.svg import SVGImage
from c3nav.mapdata.render.image.data import get_level_render_data
from c3nav.mapdata.render.image.engines.base import FillAttribs, StrokeAttribs
from c3nav.mapdata.render.image.engines.svg import SVGEngine
class SVGRenderer:
class ImageRenderer:
def __init__(self, level, minx, miny, maxx, maxy, scale=1, access_permissions=None):
self.level = level
self.minx = minx
@ -20,6 +21,9 @@ class SVGRenderer:
self.scale = scale
self.access_permissions = access_permissions
self.width = int(round((maxx - minx) * scale))
self.height = int(round((maxy - miny) * scale))
@cached_property
def bbox(self):
return box(self.minx-1, self.miny-1, self.maxx+1, self.maxy+1)
@ -61,8 +65,8 @@ class SVGRenderer:
return self.update_cache_key + ':' + self.access_cache_key
def render(self):
svg = SVGImage(bounds=((self.minx, self.miny), (self.maxx, self.maxy)),
scale=self.scale, buffer=1, background_color='#DCDCDC')
svg = SVGEngine(self.width, self.height, self.minx, self.miny,
scale=self.scale, buffer=1, background='#DCDCDC')
# add no access restriction to “unlocked“ access restrictions so lookup gets easier
unlocked_access_restrictions = self.unlocked_access_restrictions | set([None])
@ -86,15 +90,15 @@ class SVGRenderer:
# shadows are directly calculated and added by the SVGImage class
for altitudearea in geoms.altitudeareas:
svg.add_geometry(bbox.intersection(altitudearea.geometry.difference(crop_areas)),
fill_color='#eeeeee', altitude=altitudearea.altitude,
stroke_width=0.05, stroke_px=0.2, stroke_color='rgba(0, 0, 0, 0.15)')
altitude=altitudearea.altitude, fill=FillAttribs('#eeeeee'),
stroke=StrokeAttribs('rgba(0, 0, 0, 0.15)', 0.05, min_px=0.2))
for color, areas in altitudearea.colors.items():
# only select ground colors if their access restriction is unlocked
areas = tuple(area for access_restriction, area in areas.items()
if access_restriction in unlocked_access_restrictions)
if areas:
svg.add_geometry(bbox.intersection(unary_union(areas)), fill_color=color)
svg.add_geometry(bbox.intersection(unary_union(areas)), fill=FillAttribs(color))
# add walls, stroke_px makes sure that all walls are at least 1px thick on all zoom levels,
walls = None
@ -102,13 +106,13 @@ class SVGRenderer:
walls = bbox.intersection(geoms.walls.union(add_walls))
if walls is not None:
svg.add_geometry(walls, elevation=default_height, fill_color='#aaaaaa')
svg.add_geometry(walls, elevation=default_height, fill=FillAttribs('#aaaaaa'))
if not geoms.doors.is_empty:
svg.add_geometry(bbox.intersection(geoms.doors.difference(add_walls)),
fill_color='#ffffff', stroke_width=0.05, stroke_px=0.2, stroke_color='#ffffff')
svg.add_geometry(bbox.intersection(geoms.doors.difference(add_walls)), fill=FillAttribs('#ffffff'),
stroke=StrokeAttribs('#ffffff', 0.05, min_px=0.2))
if walls is not None:
svg.add_geometry(walls, stroke_width=0.05, stroke_px=0.2, stroke_color='#666666')
svg.add_geometry(walls, stroke=StrokeAttribs('#666666', 0.05, min_px=0.2))
return svg

View file

@ -0,0 +1,57 @@
import base64
import hashlib
import hmac
import time
from django.conf import settings
from django.core.cache import cache
from c3nav.mapdata.models import Level, MapUpdate
from c3nav.mapdata.models.access import AccessPermission
def get_render_level_ids(cache_key=None):
if cache_key is None:
cache_key = MapUpdate.current_cache_key()
cache_key = 'mapdata:render-level-ids:'+cache_key
levels = cache.get(cache_key, None)
if levels is None:
levels = set(Level.objects.values_list('pk', flat=True))
cache.set(cache_key, levels, 300)
return levels
def set_tile_access_cookie(request, response):
access_permissions = AccessPermission.get_for_request(request)
if access_permissions:
value = '-'.join(str(i) for i in access_permissions)+':'+str(int(time.time())+60)
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
response.set_cookie(settings.TILE_ACCESS_COOKIE_NAME, value+':'+signed, max_age=60)
else:
response.delete_cookie(settings.TILE_ACCESS_COOKIE_NAME)
def get_tile_access_cookie(request):
try:
cookie = request.COOKIES[settings.TILE_ACCESS_COOKIE_NAME]
except KeyError:
return set()
try:
access_permissions, expire, signed = cookie.split(':')
except ValueError:
return set()
value = access_permissions+':'+expire
key = hashlib.sha1(settings.SECRET_TILE_KEY.encode()).digest()
signed_verify = base64.b64encode(hmac.new(key, msg=value.encode(), digestmod=hashlib.sha256).digest()).decode()
if signed != signed_verify:
return set()
if int(expire) < time.time():
return set()
return set(int(i) for i in access_permissions.split('-'))

View file

@ -14,8 +14,8 @@ from shapely.geometry import box
from c3nav.mapdata.cache import MapHistory
from c3nav.mapdata.middleware import no_language
from c3nav.mapdata.models import Level, MapUpdate, Source
from c3nav.mapdata.render.base import get_render_level_ids, get_tile_access_cookie, set_tile_access_cookie
from c3nav.mapdata.render.svg import SVGRenderer
from c3nav.mapdata.render.image import (ImageRenderer, get_render_level_ids, get_tile_access_cookie,
set_tile_access_cookie)
@no_language()
@ -51,7 +51,7 @@ def tile(request, level, zoom, x, y, format):
access_permissions = get_tile_access_cookie(request)
# init renderer
renderer = SVGRenderer(level, minx, miny, maxx, maxy, scale=2**zoom, access_permissions=access_permissions)
renderer = ImageRenderer(level, minx, miny, maxx, maxy, scale=2**zoom, access_permissions=access_permissions)
tile_cache_key = renderer.cache_key
update_cache_key = renderer.update_cache_key

View file

@ -15,7 +15,7 @@ from c3nav.mapdata.models import Location, Source
from c3nav.mapdata.models.access import AccessPermission
from c3nav.mapdata.models.level import Level
from c3nav.mapdata.models.locations import LocationRedirect, SpecificLocation
from c3nav.mapdata.render.base import set_tile_access_cookie
from c3nav.mapdata.render.image.utils import set_tile_access_cookie
from c3nav.mapdata.utils.locations import get_location_by_slug_for_request
ctype_mapping = {