2025-summer/team-3
12
Fork 0
Code Issues Pull requests Projects Releases Activity

add heavy caching to mapdata api

Browse source
This commit is contained in:
Laura Klünder 2016-12-07 16:11:33 +01:00
parent 23da7e3605
commit 00193f7e11
22 changed files with 259 additions and 138 deletions
Download patch file Download diff file Expand all files Collapse all files

70
src/c3nav/mapdata/api.py
View file

@ -10,15 +10,17 @@ from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
from c3nav.mapdata.models import GEOMETRY_MAPITEM_TYPES, Level, Package, Source
from c3nav.mapdata.permissions import filter_queryset_by_package_access
from c3nav.mapdata.permissions import filter_queryset_by_package_access, get_unlocked_packages_names
from c3nav.mapdata.serializers.main import LevelSerializer, PackageSerializer, SourceSerializer
from c3nav.mapdata.utils.cache import (CachedReadOnlyViewSetMixin, cache_mapdata_api_response, get_levels_cached,
get_packages_cached)
class GeometryTypeViewSet(ViewSet):
"""
Lists all geometry types.
"""
@cache_mapdata_api_response()
def list(self, request):
return Response([
OrderedDict((
@ -32,41 +34,53 @@ class GeometryTypeViewSet(ViewSet):
class GeometryViewSet(ViewSet):
"""
List all geometries.
You can filter by adding one or more level, package, type or name GET parameters.
You can filter by adding a level GET parameter or one or more package or type GET parameters.
"""
def list(self, request):
types = request.GET.getlist('type')
types = set(request.GET.getlist('type'))
valid_types = list(GEOMETRY_MAPITEM_TYPES.keys())
if not types:
types = valid_types
else:
types = [t for t in types if t in valid_types]
types = [t for t in valid_types if t in types]
levels = request.GET.getlist('level')
packages = request.GET.getlist('package')
names = request.GET.getlist('name')
level = None
if 'level' in request.GET:
levels_cached = get_levels_cached()
level_name = request.GET['level']
if level_name in levels_cached:
level = levels_cached[level_name]
if levels:
levels = tuple(Level.objects.filter(name__in=levels))
if packages:
packages = tuple(Package.objects.filter(name__in=packages))
packages_cached = get_packages_cached()
package_names = set(request.GET.getlist('package')) & set(get_unlocked_packages_names(request))
packages = [packages_cached[name] for name in package_names if name in packages_cached]
if len(packages) == len(packages_cached):
packages = []
package_ids = sorted([package.id for package in packages])
cache_key = '__'.join((
','.join([str(i) for i in types]),
str(level.id) if level is not None else '',
','.join([str(i) for i in package_ids]),
))
return self._list(request, types=types, level=level, packages=packages, add_cache_key=cache_key)
@cache_mapdata_api_response()
def _list(self, request, types, level, packages):
results = []
for t in types:
mapitemtype = GEOMETRY_MAPITEM_TYPES[t]
queryset = mapitemtype.objects.all()
if packages:
queryset = queryset.filter(package__in=packages)
if levels:
if level:
if hasattr(mapitemtype, 'level'):
queryset = queryset.filter(level__in=levels)
queryset = queryset.filter(level=level)
elif hasattr(mapitemtype, 'levels'):
queryset = queryset.filter(levels__in=levels)
queryset = queryset.filter(levels=level)
else:
queryset = queryset.none()
if names:
queryset = queryset.filter(name__in=names)
queryset = filter_queryset_by_package_access(request, queryset)
queryset = queryset.order_by('name')
@ -83,7 +97,7 @@ class GeometryViewSet(ViewSet):
return Response(results)
class PackageViewSet(ReadOnlyModelViewSet):
class PackageViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
Retrieve packages the map consists of.
"""
@ -91,13 +105,10 @@ class PackageViewSet(ReadOnlyModelViewSet):
serializer_class = PackageSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
filter_fields = ('name', 'depends')
ordering_fields = ('name',)
ordering = ('name',)
search_fields = ('name',)
class LevelViewSet(ReadOnlyModelViewSet):
class LevelViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
List and retrieve levels.
"""
@ -105,13 +116,10 @@ class LevelViewSet(ReadOnlyModelViewSet):
serializer_class = LevelSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
filter_fields = ('altitude', 'package')
ordering_fields = ('altitude', 'package')
ordering = ('altitude',)
search_fields = ('name',)
class SourceViewSet(ReadOnlyModelViewSet):
class SourceViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
"""
List and retrieve source images (to use as a drafts).
"""
@ -119,16 +127,18 @@ class SourceViewSet(ReadOnlyModelViewSet):
serializer_class = SourceSerializer
lookup_field = 'name'
lookup_value_regex = '[^/]+'
filter_fields = ('package',)
ordering_fields = ('name', 'package')
ordering = ('name',)
search_fields = ('name',)
include_package_access = True
def get_queryset(self):
return filter_queryset_by_package_access(self.request, super().get_queryset())
@detail_route(methods=['get'])
def image(self, request, name=None):
return self._image(request, name=name, add_cache_key=self._get_add_cache_key(request))
@cache_mapdata_api_response()
def _image(self, request, name=None):
source = self.get_object()
response = HttpResponse(content_type=mimetypes.guess_type(source.name)[0])
image_path = os.path.join(settings.MAP_ROOT, source.package.directory, 'sources', source.name)

43
src/c3nav/mapdata/cache.py
View file

@ -1,43 +0,0 @@
import base64
from django.core.cache import cache
from django.template.response import SimpleTemplateResponse
from django.utils.cache import patch_vary_headers
from c3nav.mapdata.permissions import get_unlocked_packages
class CachedViewSetMixin:
def get_cache_key(self, request):
cache_key = ('api__' + ('OPTIONS' if request.method == 'OPTIONS' else 'GET') + '_' +
base64.b64encode(self.get_cache_params(request).encode()).decode() + '_' +
request.path + '?' + request.META['QUERY_STRING'])
return cache_key
def get_cache_params(self, request):
return request.META.get('HTTP_ACCEPT', '')
def dispatch(self, request, *args, **kwargs):
do_cache = request.method in ('GET', 'HEAD', 'OPTIONS')
if do_cache:
cache_key = self.get_cache_key(request)
if cache_key in cache:
return cache.get(cache_key)
response = super().dispatch(request, *args, **kwargs)
patch_vary_headers(response, ['Cookie'])
if do_cache:
if isinstance(response, SimpleTemplateResponse):
response.render()
cache.set(cache_key, response, 60)
return response
@property
def default_response_headers(self):
headers = super().default_response_headers
headers['Vary'] += ', Cookie'
return headers
class AccessCachedViewSetMixin(CachedViewSetMixin):
def get_cache_params(self, request):
return super().get_cache_params(request) + '___' + '___'.join(get_unlocked_packages(request))

3
src/c3nav/mapdata/fields.py
View file

@ -6,7 +6,8 @@ from shapely import validation
from shapely.geometry import mapping, shape
from shapely.geometry.base import BaseGeometry
from c3nav.mapdata.utils import clean_geometry, format_geojson
from c3nav.mapdata.utils.geometry import clean_geometry
from c3nav.mapdata.utils.json import format_geojson
def validate_geometry(geometry):

33
src/c3nav/mapdata/lastupdate.py Normal file
View file

@ -0,0 +1,33 @@
import os
import pickle
from contextlib import contextmanager
from django.conf import settings
from django.utils import timezone
last_mapdata_update_filename = os.path.join(settings.DATA_DIR, 'last_mapdata_update')
last_mapdata_update_decorator_depth = 0
def get_last_mapdata_update(default_now=False):
try:
with open(last_mapdata_update_filename, 'rb') as f:
return pickle.load(f)
except:
return timezone.now() if default_now else None
@contextmanager
def set_last_mapdata_update():
global last_mapdata_update_decorator_depth
if last_mapdata_update_decorator_depth == 0:
try:
os.remove(last_mapdata_update_filename)
except:
pass
last_mapdata_update_decorator_depth += 1
yield
last_mapdata_update_decorator_depth -= 1
if last_mapdata_update_decorator_depth == 0:
with open(last_mapdata_update_filename, 'wb') as f:
pickle.dump(timezone.now(), f)

11
src/c3nav/mapdata/management/commands/clearmapcache.py Normal file
View file

@ -0,0 +1,11 @@
from django.core.management.base import BaseCommand
from c3nav.mapdata.lastupdate import set_last_mapdata_update
class Command(BaseCommand):
help = 'Clear the map cache (set last updated to now)'
def handle(self, *args, **options):
with set_last_mapdata_update():
pass

12
src/c3nav/mapdata/management/commands/loadmap.py
View file

@ -1,6 +1,7 @@
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from c3nav.mapdata.lastupdate import set_last_mapdata_update
from c3nav.mapdata.packageio import MapdataReader
@ -15,8 +16,9 @@ class Command(BaseCommand):
reader = MapdataReader()
reader.read_packages()
with transaction.atomic():
reader.apply_to_db()
print()
if not options['yes'] and input('Confirm (y/N): ') != 'y':
raise CommandError('Aborted.')
with set_last_mapdata_update():
with transaction.atomic():
reader.apply_to_db()
print()
if not options['yes'] and input('Confirm (y/N): ') != 'y':
raise CommandError('Aborted.')

6
src/c3nav/mapdata/models/base.py
View file

@ -4,6 +4,8 @@ from django.db import models
from django.db.models.base import ModelBase
from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.lastupdate import set_last_mapdata_update
MAPITEM_TYPES = OrderedDict()
@ -40,5 +42,9 @@ class MapItem(models.Model, metaclass=MapItemMeta):
def tofile(self):
return OrderedDict()
def save(self, *args, **kwargs):
with set_last_mapdata_update():
super().save(*args, **kwargs)
class Meta:
abstract = True

2
src/c3nav/mapdata/models/geometry.py
View file

@ -8,7 +8,7 @@ from shapely.geometry.geo import mapping, shape
from c3nav.mapdata.fields import GeometryField
from c3nav.mapdata.models import Elevator
from c3nav.mapdata.models.base import MapItem, MapItemMeta
from c3nav.mapdata.utils import format_geojson
from c3nav.mapdata.utils.json import format_geojson
GEOMETRY_MAPITEM_TYPES = OrderedDict()

6
src/c3nav/mapdata/models/package.py
View file

@ -4,6 +4,8 @@ from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from c3nav.mapdata.lastupdate import set_last_mapdata_update
class Package(models.Model):
"""
@ -93,5 +95,9 @@ class Package(models.Model):
return data
def save(self, *args, **kwargs):
with set_last_mapdata_update():
super().save(*args, **kwargs)
def __str__(self):
return self.name

2
src/c3nav/mapdata/packageio/write.py
View file

@ -9,7 +9,7 @@ from django.utils import timezone
from c3nav.mapdata.models import Package
from c3nav.mapdata.packageio.const import ordered_models
from c3nav.mapdata.utils import json_encoder_reindent
from c3nav.mapdata.utils.json import json_encoder_reindent
class MapdataWriter:

17
src/c3nav/mapdata/permissions.py
View file

@ -3,16 +3,23 @@ from django.utils.translation import ugettext_lazy as _
from rest_framework.exceptions import PermissionDenied
from rest_framework.permissions import BasePermission
from c3nav.mapdata.models import Package, Source
from c3nav.mapdata.models import Source
from c3nav.mapdata.utils.cache import get_packages_cached
def get_unlocked_packages_names(request):
def get_unlocked_packages_names(request, packages_cached=None):
if packages_cached is None:
packages_cached = get_packages_cached()
if settings.DIRECT_EDITING:
return packages_cached.keys()
return set(settings.PUBLIC_PACKAGES) | set(request.session.get('unlocked_packages', ()))
def get_unlocked_packages(request):
names = get_unlocked_packages_names(request)
return tuple(Package.objects.filter(name__in=names))
def get_unlocked_packages(request, packages_cached=None):
if packages_cached is None:
packages_cached = get_packages_cached()
names = get_unlocked_packages_names(request, packages_cached=packages_cached)
return tuple(packages_cached[name] for name in names if name in packages_cached)
def can_access_package(request, package):

2
src/c3nav/mapdata/serializers/fields.py
View file

@ -3,7 +3,7 @@ from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from shapely.geometry import mapping, shape
from c3nav.mapdata.utils import format_geojson
from c3nav.mapdata.utils.json import format_geojson
class GeometryField(serializers.DictField):

0
src/c3nav/mapdata/utils/__init__.py Normal file
View file

99
src/c3nav/mapdata/utils/cache.py Normal file
View file

@ -0,0 +1,99 @@
from calendar import timegm
from functools import wraps
from django.core.cache import cache
from django.utils.http import http_date
from rest_framework.response import Response as APIResponse
from rest_framework.views import APIView
from c3nav.mapdata.lastupdate import get_last_mapdata_update
def cache_result(cache_key, timeout=900):
def decorator(func):
@wraps(func)
def inner(*args, **kwargs):
last_update = get_last_mapdata_update()
if last_update is None:
return func(*args, **kwargs)
result = cache.get(cache_key)
if not result:
result = func(*args, **kwargs)
cache.set(cache_key, result, timeout)
return result
return inner
return decorator
def cache_mapdata_api_response(timeout=900):
def decorator(func):
@wraps(func)
def inner(self, request, *args, add_cache_key=None, **kwargs):
last_update = get_last_mapdata_update()
if last_update is None:
return func(self, request, *args, **kwargs)
cache_key = '__'.join((
'c3nav__mapdata__api',
last_update.isoformat(),
add_cache_key if add_cache_key is not None else '',
request.accepted_renderer.format if isinstance(self, APIView) else '',
request.path,
))
response = cache.get(cache_key)
if not response:
response = func(self, request, *args, **kwargs)
response['Last-Modifed'] = http_date(timegm(last_update.utctimetuple()))
if isinstance(response, APIResponse):
response = self.finalize_response(request, response, *args, **kwargs)
response.render()
if response.status_code < 400:
cache.set(cache_key, response, timeout)
return response
return inner
return decorator
class CachedReadOnlyViewSetMixin():
include_package_access = False
def _get_unlocked_packages_ids(self, request):
from c3nav.mapdata.permissions import get_unlocked_packages
return ','.join(str(i) for i in sorted(package.id for package in get_unlocked_packages(request)))
def _get_add_cache_key(self, request, add_cache_key=''):
cache_key = add_cache_key
if self.include_package_access:
cache_key += '__'+self._get_unlocked_packages_ids(request)
return cache_key
def list(self, request, *args, **kwargs):
kwargs['add_cache_key'] = self._get_add_cache_key(request, kwargs.get('add_cache_key', ''))
return self._list(request, *args, **kwargs)
@cache_mapdata_api_response()
def _list(self, request, *args, **kwargs):
return super().list(request, *args, **kwargs)
def retrieve(self, request, *args, **kwargs):
kwargs['add_cache_key'] = self._get_add_cache_key(request, kwargs.get('add_cache_key', ''))
return self._retrieve(request, *args, **kwargs)
@cache_mapdata_api_response()
def _retrieve(self, request, *args, **kwargs):
return super().retrieve(request, *args, **kwargs)
@cache_result('c3nav__mapdata__levels')
def get_levels_cached():
from c3nav.mapdata.models import Level
return {level.name: level for level in Level.objects.all()}
@cache_result('c3nav__mapdata__packages')
def get_packages_cached():
from c3nav.mapdata.models import Package
return {package.name: package for package in Package.objects.all()}

32
src/c3nav/mapdata/utils/geometry.py Normal file
View file

@ -0,0 +1,32 @@
from shapely.geometry import Polygon
def clean_geometry(geometry):
"""
if the given geometry is a Polygon and invalid, try to make it valid if it results in a Polygon (not MultiPolygon)
"""
if geometry.is_valid:
return geometry
if isinstance(geometry, Polygon):
p = Polygon(list(geometry.exterior.coords))
for interior in geometry.interiors:
p = p.difference(Polygon(list(interior.coords)))
if isinstance(p, Polygon) and p.is_valid:
return p
return geometry
def assert_multipolygon(geometry):
"""
given a Polygon or a MultiPolygon, return a list of Polygons
:param geometry: a Polygon or a MultiPolygon
:return: a list of Polygons
"""
if isinstance(geometry, Polygon):
polygons = [geometry]
else:
polygons = geometry.geoms
return polygons

33
src/c3nav/mapdata/utils.py → src/c3nav/mapdata/utils/json.py
View file

@ -1,8 +1,6 @@
import json
from collections import OrderedDict
from shapely.geometry import Polygon
def _preencode(data, magic_marker, in_coords=False):
if isinstance(data, dict):
@ -47,34 +45,3 @@ def round_coordinates(data):
return tuple(round_coordinates(item) for item in data)
else:
return round(data, 2)
def clean_geometry(geometry):
"""
if the given geometry is a Polygon and invalid, try to make it valid if it results in a Polygon (not MultiPolygon)
"""
if geometry.is_valid:
return geometry
if isinstance(geometry, Polygon):
p = Polygon(list(geometry.exterior.coords))
for interior in geometry.interiors:
p = p.difference(Polygon(list(interior.coords)))
if isinstance(p, Polygon) and p.is_valid:
return p
return geometry
def assert_multipolygon(geometry):
"""
given a Polygon or a MultiPolygon, return a list of Polygons
:param geometry: a Polygon or a MultiPolygon
:return: a list of Polygons
"""
if isinstance(geometry, Polygon):
polygons = [geometry]
else:
polygons = geometry.geoms
return polygons