add heavy caching to mapdata api

src/c3nav/api/__init__.py

@@ -2,7 +2,7 @@ from functools import wraps
 
 from rest_framework.renderers import JSONRenderer
 
-from c3nav.mapdata.utils import json_encoder_reindent
+from c3nav.mapdata.utils.json import json_encoder_reindent
 
 orig_render = JSONRenderer.render
 
@@ -11,11 +11,11 @@ orig_render = JSONRenderer.render
 def nicer_renderer(self, data, accepted_media_type=None, renderer_context=None):
     if self.get_indent(accepted_media_type, renderer_context) is None:
         return orig_render(self, data, accepted_media_type, renderer_context)
-    shorten = isinstance(data, (list, tuple)) and len(data) > 2
+    shorten = isinstance(data, (list, tuple)) and len(data) > 5
     orig_len = None
     if shorten:
-        orig_len = len(data)-2
+        orig_len = len(data)-5
-        data = data[:2]
+        data = data[:5]
     result = json_encoder_reindent(lambda d: orig_render(self, d, accepted_media_type, renderer_context), data)
     if shorten:
         result = (result[:-2] +

src/c3nav/mapdata/api.py

@@ -10,15 +10,17 @@ from rest_framework.response import Response
 from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
 
 from c3nav.mapdata.models import GEOMETRY_MAPITEM_TYPES, Level, Package, Source
-from c3nav.mapdata.permissions import filter_queryset_by_package_access
+from c3nav.mapdata.permissions import filter_queryset_by_package_access, get_unlocked_packages_names
 from c3nav.mapdata.serializers.main import LevelSerializer, PackageSerializer, SourceSerializer
+from c3nav.mapdata.utils.cache import (CachedReadOnlyViewSetMixin, cache_mapdata_api_response, get_levels_cached,
+                                       get_packages_cached)
 
 
 class GeometryTypeViewSet(ViewSet):
     """
     Lists all geometry types.
     """
-
+    @cache_mapdata_api_response()
     def list(self, request):
         return Response([
             OrderedDict((
@@ -32,41 +34,53 @@ class GeometryTypeViewSet(ViewSet):
 class GeometryViewSet(ViewSet):
     """
     List all geometries.
-    You can filter by adding one or more level, package, type or name GET parameters.
+    You can filter by adding a level GET parameter or one or more package or type GET parameters.
     """
-
     def list(self, request):
-        types = request.GET.getlist('type')
+        types = set(request.GET.getlist('type'))
         valid_types = list(GEOMETRY_MAPITEM_TYPES.keys())
         if not types:
             types = valid_types
         else:
-            types = [t for t in types if t in valid_types]
+            types = [t for t in valid_types if t in types]
 
-        levels = request.GET.getlist('level')
+        level = None
-        packages = request.GET.getlist('package')
+        if 'level' in request.GET:
-        names = request.GET.getlist('name')
+            levels_cached = get_levels_cached()
+            level_name = request.GET['level']
+            if level_name in levels_cached:
+                level = levels_cached[level_name]
 
-        if levels:
+        packages_cached = get_packages_cached()
-            levels = tuple(Level.objects.filter(name__in=levels))
+        package_names = set(request.GET.getlist('package')) & set(get_unlocked_packages_names(request))
-        if packages:
+        packages = [packages_cached[name] for name in package_names if name in packages_cached]
-            packages = tuple(Package.objects.filter(name__in=packages))
+        if len(packages) == len(packages_cached):
+            packages = []
+        package_ids = sorted([package.id for package in packages])
 
+        cache_key = '__'.join((
+            ','.join([str(i) for i in types]),
+            str(level.id) if level is not None else '',
+            ','.join([str(i) for i in package_ids]),
+        ))
+
+        return self._list(request, types=types, level=level, packages=packages, add_cache_key=cache_key)
+
+    @cache_mapdata_api_response()
+    def _list(self, request, types, level, packages):
         results = []
         for t in types:
             mapitemtype = GEOMETRY_MAPITEM_TYPES[t]
             queryset = mapitemtype.objects.all()
             if packages:
                 queryset = queryset.filter(package__in=packages)
-            if levels:
+            if level:
                 if hasattr(mapitemtype, 'level'):
-                    queryset = queryset.filter(level__in=levels)
+                    queryset = queryset.filter(level=level)
                 elif hasattr(mapitemtype, 'levels'):
-                    queryset = queryset.filter(levels__in=levels)
+                    queryset = queryset.filter(levels=level)
                 else:
                     queryset = queryset.none()
-            if names:
-                queryset = queryset.filter(name__in=names)
             queryset = filter_queryset_by_package_access(request, queryset)
             queryset = queryset.order_by('name')
 
@@ -83,7 +97,7 @@ class GeometryViewSet(ViewSet):
         return Response(results)
 
 
-class PackageViewSet(ReadOnlyModelViewSet):
+class PackageViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
     """
     Retrieve packages the map consists of.
     """
@@ -91,13 +105,10 @@ class PackageViewSet(ReadOnlyModelViewSet):
     serializer_class = PackageSerializer
     lookup_field = 'name'
     lookup_value_regex = '[^/]+'
-    filter_fields = ('name', 'depends')
-    ordering_fields = ('name',)
     ordering = ('name',)
-    search_fields = ('name',)
 
 
-class LevelViewSet(ReadOnlyModelViewSet):
+class LevelViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
     """
     List and retrieve levels.
     """
@@ -105,13 +116,10 @@ class LevelViewSet(ReadOnlyModelViewSet):
     serializer_class = LevelSerializer
     lookup_field = 'name'
     lookup_value_regex = '[^/]+'
-    filter_fields = ('altitude', 'package')
-    ordering_fields = ('altitude', 'package')
     ordering = ('altitude',)
-    search_fields = ('name',)
 
 
-class SourceViewSet(ReadOnlyModelViewSet):
+class SourceViewSet(CachedReadOnlyViewSetMixin, ReadOnlyModelViewSet):
     """
     List and retrieve source images (to use as a drafts).
     """
@@ -119,16 +127,18 @@ class SourceViewSet(ReadOnlyModelViewSet):
     serializer_class = SourceSerializer
     lookup_field = 'name'
     lookup_value_regex = '[^/]+'
-    filter_fields = ('package',)
-    ordering_fields = ('name', 'package')
     ordering = ('name',)
-    search_fields = ('name',)
+    include_package_access = True
 
     def get_queryset(self):
         return filter_queryset_by_package_access(self.request, super().get_queryset())
 
     @detail_route(methods=['get'])
     def image(self, request, name=None):
+        return self._image(request, name=name, add_cache_key=self._get_add_cache_key(request))
+
+    @cache_mapdata_api_response()
+    def _image(self, request, name=None):
         source = self.get_object()
         response = HttpResponse(content_type=mimetypes.guess_type(source.name)[0])
         image_path = os.path.join(settings.MAP_ROOT, source.package.directory, 'sources', source.name)

src/c3nav/mapdata/cache.py

@@ -1,43 +0,0 @@
-import base64
-
-from django.core.cache import cache
-from django.template.response import SimpleTemplateResponse
-from django.utils.cache import patch_vary_headers
-
-from c3nav.mapdata.permissions import get_unlocked_packages
-
-
-class CachedViewSetMixin:
-    def get_cache_key(self, request):
-        cache_key = ('api__' + ('OPTIONS' if request.method == 'OPTIONS' else 'GET') + '_' +
-                     base64.b64encode(self.get_cache_params(request).encode()).decode() + '_' +
-                     request.path + '?' + request.META['QUERY_STRING'])
-        return cache_key
-
-    def get_cache_params(self, request):
-        return request.META.get('HTTP_ACCEPT', '')
-
-    def dispatch(self, request, *args, **kwargs):
-        do_cache = request.method in ('GET', 'HEAD', 'OPTIONS')
-        if do_cache:
-            cache_key = self.get_cache_key(request)
-            if cache_key in cache:
-                return cache.get(cache_key)
-        response = super().dispatch(request, *args, **kwargs)
-        patch_vary_headers(response, ['Cookie'])
-        if do_cache:
-            if isinstance(response, SimpleTemplateResponse):
-                response.render()
-            cache.set(cache_key, response, 60)
-        return response
-
-    @property
-    def default_response_headers(self):
-        headers = super().default_response_headers
-        headers['Vary'] += ', Cookie'
-        return headers
-
-
-class AccessCachedViewSetMixin(CachedViewSetMixin):
-    def get_cache_params(self, request):
-        return super().get_cache_params(request) + '___' + '___'.join(get_unlocked_packages(request))

src/c3nav/mapdata/fields.py

@@ -6,7 +6,8 @@ from shapely import validation
 from shapely.geometry import mapping, shape
 from shapely.geometry.base import BaseGeometry
 
-from c3nav.mapdata.utils import clean_geometry, format_geojson
+from c3nav.mapdata.utils.geometry import clean_geometry
+from c3nav.mapdata.utils.json import format_geojson
 
 
 def validate_geometry(geometry):

src/c3nav/mapdata/lastupdate.py

@@ -0,0 +1,33 @@
+import os
+import pickle
+from contextlib import contextmanager
+
+from django.conf import settings
+from django.utils import timezone
+
+last_mapdata_update_filename = os.path.join(settings.DATA_DIR, 'last_mapdata_update')
+last_mapdata_update_decorator_depth = 0
+
+
+def get_last_mapdata_update(default_now=False):
+    try:
+        with open(last_mapdata_update_filename, 'rb') as f:
+            return pickle.load(f)
+    except:
+        return timezone.now() if default_now else None
+
+
+@contextmanager
+def set_last_mapdata_update():
+    global last_mapdata_update_decorator_depth
+    if last_mapdata_update_decorator_depth == 0:
+        try:
+            os.remove(last_mapdata_update_filename)
+        except:
+            pass
+    last_mapdata_update_decorator_depth += 1
+    yield
+    last_mapdata_update_decorator_depth -= 1
+    if last_mapdata_update_decorator_depth == 0:
+        with open(last_mapdata_update_filename, 'wb') as f:
+            pickle.dump(timezone.now(), f)

src/c3nav/mapdata/management/commands/clearmapcache.py

@@ -0,0 +1,11 @@
+from django.core.management.base import BaseCommand
+
+from c3nav.mapdata.lastupdate import set_last_mapdata_update
+
+
+class Command(BaseCommand):
+    help = 'Clear the map cache (set last updated to now)'
+
+    def handle(self, *args, **options):
+        with set_last_mapdata_update():
+            pass

src/c3nav/mapdata/management/commands/loadmap.py

@@ -1,6 +1,7 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
+from c3nav.mapdata.lastupdate import set_last_mapdata_update
 from c3nav.mapdata.packageio import MapdataReader
 
 
@@ -15,6 +16,7 @@ class Command(BaseCommand):
         reader = MapdataReader()
         reader.read_packages()
 
+        with set_last_mapdata_update():
             with transaction.atomic():
                 reader.apply_to_db()
                 print()

src/c3nav/mapdata/models/base.py

@@ -4,6 +4,8 @@ from django.db import models
 from django.db.models.base import ModelBase
 from django.utils.translation import ugettext_lazy as _
 
+from c3nav.mapdata.lastupdate import set_last_mapdata_update
+
 MAPITEM_TYPES = OrderedDict()
 
 
@@ -40,5 +42,9 @@ class MapItem(models.Model, metaclass=MapItemMeta):
     def tofile(self):
         return OrderedDict()
 
+    def save(self, *args, **kwargs):
+        with set_last_mapdata_update():
+            super().save(*args, **kwargs)
+
     class Meta:
         abstract = True

src/c3nav/mapdata/models/geometry.py

@@ -8,7 +8,7 @@ from shapely.geometry.geo import mapping, shape
 from c3nav.mapdata.fields import GeometryField
 from c3nav.mapdata.models import Elevator
 from c3nav.mapdata.models.base import MapItem, MapItemMeta
-from c3nav.mapdata.utils import format_geojson
+from c3nav.mapdata.utils.json import format_geojson
 
 GEOMETRY_MAPITEM_TYPES = OrderedDict()
 

src/c3nav/mapdata/models/package.py

@@ -4,6 +4,8 @@ from django.conf import settings
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
+from c3nav.mapdata.lastupdate import set_last_mapdata_update
+
 
 class Package(models.Model):
     """
@@ -93,5 +95,9 @@ class Package(models.Model):
 
         return data
 
+    def save(self, *args, **kwargs):
+        with set_last_mapdata_update():
+            super().save(*args, **kwargs)
+
     def __str__(self):
         return self.name

src/c3nav/mapdata/packageio/write.py

@@ -9,7 +9,7 @@ from django.utils import timezone
 
 from c3nav.mapdata.models import Package
 from c3nav.mapdata.packageio.const import ordered_models
-from c3nav.mapdata.utils import json_encoder_reindent
+from c3nav.mapdata.utils.json import json_encoder_reindent
 
 
 class MapdataWriter:

src/c3nav/mapdata/permissions.py

@@ -3,16 +3,23 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework.exceptions import PermissionDenied
 from rest_framework.permissions import BasePermission
 
-from c3nav.mapdata.models import Package, Source
+from c3nav.mapdata.models import Source
+from c3nav.mapdata.utils.cache import get_packages_cached
 
 
-def get_unlocked_packages_names(request):
+def get_unlocked_packages_names(request, packages_cached=None):
+    if packages_cached is None:
+        packages_cached = get_packages_cached()
+    if settings.DIRECT_EDITING:
+        return packages_cached.keys()
     return set(settings.PUBLIC_PACKAGES) | set(request.session.get('unlocked_packages', ()))
 
 
-def get_unlocked_packages(request):
+def get_unlocked_packages(request, packages_cached=None):
-    names = get_unlocked_packages_names(request)
+    if packages_cached is None:
-    return tuple(Package.objects.filter(name__in=names))
+        packages_cached = get_packages_cached()
+    names = get_unlocked_packages_names(request, packages_cached=packages_cached)
+    return tuple(packages_cached[name] for name in names if name in packages_cached)
 
 
 def can_access_package(request, package):

src/c3nav/mapdata/serializers/fields.py

@@ -3,7 +3,7 @@ from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 from shapely.geometry import mapping, shape
 
-from c3nav.mapdata.utils import format_geojson
+from c3nav.mapdata.utils.json import format_geojson
 
 
 class GeometryField(serializers.DictField):

src/c3nav/mapdata/utils/cache.py

@@ -0,0 +1,99 @@
+from calendar import timegm
+from functools import wraps
+
+from django.core.cache import cache
+from django.utils.http import http_date
+from rest_framework.response import Response as APIResponse
+from rest_framework.views import APIView
+
+from c3nav.mapdata.lastupdate import get_last_mapdata_update
+
+
+def cache_result(cache_key, timeout=900):
+    def decorator(func):
+        @wraps(func)
+        def inner(*args, **kwargs):
+            last_update = get_last_mapdata_update()
+            if last_update is None:
+                return func(*args, **kwargs)
+
+            result = cache.get(cache_key)
+            if not result:
+                result = func(*args, **kwargs)
+                cache.set(cache_key, result, timeout)
+            return result
+        return inner
+    return decorator
+
+
+def cache_mapdata_api_response(timeout=900):
+    def decorator(func):
+        @wraps(func)
+        def inner(self, request, *args, add_cache_key=None, **kwargs):
+            last_update = get_last_mapdata_update()
+            if last_update is None:
+                return func(self, request, *args, **kwargs)
+
+            cache_key = '__'.join((
+                'c3nav__mapdata__api',
+                last_update.isoformat(),
+                add_cache_key if add_cache_key is not None else '',
+                request.accepted_renderer.format if isinstance(self, APIView) else '',
+                request.path,
+            ))
+
+            response = cache.get(cache_key)
+            if not response:
+                response = func(self, request, *args, **kwargs)
+                response['Last-Modifed'] = http_date(timegm(last_update.utctimetuple()))
+                if isinstance(response, APIResponse):
+                    response = self.finalize_response(request, response, *args, **kwargs)
+                    response.render()
+                if response.status_code < 400:
+                    cache.set(cache_key, response, timeout)
+
+            return response
+        return inner
+    return decorator
+
+
+class CachedReadOnlyViewSetMixin():
+    include_package_access = False
+
+    def _get_unlocked_packages_ids(self, request):
+        from c3nav.mapdata.permissions import get_unlocked_packages
+        return ','.join(str(i) for i in sorted(package.id for package in get_unlocked_packages(request)))
+
+    def _get_add_cache_key(self, request, add_cache_key=''):
+        cache_key = add_cache_key
+        if self.include_package_access:
+            cache_key += '__'+self._get_unlocked_packages_ids(request)
+        return cache_key
+
+    def list(self, request, *args, **kwargs):
+        kwargs['add_cache_key'] = self._get_add_cache_key(request, kwargs.get('add_cache_key', ''))
+        return self._list(request, *args, **kwargs)
+
+    @cache_mapdata_api_response()
+    def _list(self, request, *args, **kwargs):
+        return super().list(request, *args, **kwargs)
+
+    def retrieve(self, request, *args, **kwargs):
+        kwargs['add_cache_key'] = self._get_add_cache_key(request, kwargs.get('add_cache_key', ''))
+        return self._retrieve(request, *args, **kwargs)
+
+    @cache_mapdata_api_response()
+    def _retrieve(self, request, *args, **kwargs):
+        return super().retrieve(request, *args, **kwargs)
+
+
+@cache_result('c3nav__mapdata__levels')
+def get_levels_cached():
+    from c3nav.mapdata.models import Level
+    return {level.name: level for level in Level.objects.all()}
+
+
+@cache_result('c3nav__mapdata__packages')
+def get_packages_cached():
+    from c3nav.mapdata.models import Package
+    return {package.name: package for package in Package.objects.all()}

src/c3nav/mapdata/utils/geometry.py

@@ -0,0 +1,32 @@
+from shapely.geometry import Polygon
+
+
+def clean_geometry(geometry):
+    """
+    if the given geometry is a Polygon and invalid, try to make it valid if it results in a Polygon (not MultiPolygon)
+    """
+    if geometry.is_valid:
+        return geometry
+
+    if isinstance(geometry, Polygon):
+        p = Polygon(list(geometry.exterior.coords))
+        for interior in geometry.interiors:
+            p = p.difference(Polygon(list(interior.coords)))
+
+        if isinstance(p, Polygon) and p.is_valid:
+            return p
+
+    return geometry
+
+
+def assert_multipolygon(geometry):
+    """
+    given a Polygon or a MultiPolygon, return a list of Polygons
+    :param geometry: a Polygon or a MultiPolygon
+    :return: a list of Polygons
+    """
+    if isinstance(geometry, Polygon):
+        polygons = [geometry]
+    else:
+        polygons = geometry.geoms
+    return polygons

src/c3nav/mapdata/utils/json.py

@@ -1,8 +1,6 @@
 import json
 from collections import OrderedDict
 
-from shapely.geometry import Polygon
-
 
 def _preencode(data, magic_marker, in_coords=False):
     if isinstance(data, dict):
@@ -47,34 +45,3 @@ def round_coordinates(data):
         return tuple(round_coordinates(item) for item in data)
     else:
         return round(data, 2)
-
-
-def clean_geometry(geometry):
-    """
-    if the given geometry is a Polygon and invalid, try to make it valid if it results in a Polygon (not MultiPolygon)
-    """
-    if geometry.is_valid:
-        return geometry
-
-    if isinstance(geometry, Polygon):
-        p = Polygon(list(geometry.exterior.coords))
-        for interior in geometry.interiors:
-            p = p.difference(Polygon(list(interior.coords)))
-
-        if isinstance(p, Polygon) and p.is_valid:
-            return p
-
-    return geometry
-
-
-def assert_multipolygon(geometry):
-    """
-    given a Polygon or a MultiPolygon, return a list of Polygons
-    :param geometry: a Polygon or a MultiPolygon
-    :return: a list of Polygons
-    """
-    if isinstance(geometry, Polygon):
-        polygons = [geometry]
-    else:
-        polygons = geometry.geoms
-    return polygons

src/c3nav/routing/level.py

@@ -4,7 +4,7 @@ from django.conf import settings
 from PIL import Image, ImageDraw
 from shapely.geometry import JOIN_STYLE
 
-from c3nav.mapdata.utils import assert_multipolygon
+from c3nav.mapdata.utils.geometry import assert_multipolygon
 from c3nav.routing.point import GraphPoint
 from c3nav.routing.room import GraphRoom
 from c3nav.routing.utils.base import get_nearest_point

src/c3nav/routing/room.py

@@ -4,7 +4,7 @@ import numpy as np
 from matplotlib.path import Path
 from shapely.geometry import JOIN_STYLE, LineString
 
-from c3nav.mapdata.utils import assert_multipolygon
+from c3nav.mapdata.utils.geometry import assert_multipolygon
 from c3nav.routing.point import GraphPoint
 from c3nav.routing.router import Router
 from c3nav.routing.utils.coords import get_coords_angles

src/c3nav/routing/utils/base.py

@@ -1,6 +1,6 @@
 from shapely.geometry import Polygon
 
-from c3nav.mapdata.utils import assert_multipolygon
+from c3nav.mapdata.utils.geometry import assert_multipolygon
 
 
 def get_nearest_point(polygon, point):

src/c3nav/routing/utils/mpl.py

@@ -1,6 +1,6 @@
 from matplotlib.path import Path
 
-from c3nav.mapdata.utils import assert_multipolygon
+from c3nav.mapdata.utils.geometry import assert_multipolygon
 
 
 def polygon_to_mpl_paths(polygon):

src/c3nav/settings.py

@@ -187,16 +187,6 @@ REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': (
         'c3nav.api.authentication.ForceCSRFCheckSessionAuthentication',
     ),
-    'DEFAULT_PERMISSION_CLASSES': (
-        'c3nav.mapdata.permissions.LockedMapFeatures',
-    ),
-    'DEFAULT_FILTER_BACKENDS': (
-        'rest_framework.filters.DjangoFilterBackend',
-        'rest_framework.filters.OrderingFilter',
-        'rest_framework.filters.SearchFilter',
-    ),
-    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
-    # 'PAGE_SIZE': 50
 }
 
 LOCALE_PATHS = (