save level render data into a file instead of pickling it into the db

src/c3nav/mapdata/management/commands/clearmapcache.py

@@ -17,9 +17,9 @@ class Command(BaseCommand):
         MapUpdate.objects.create(type='management')
 
         if options['include_history']:
-            for file in os.listdir(settings.CACHE_ROOT):
-                if file.startswith('level_'):
-                    os.remove(os.path.join(settings.CACHE_ROOT, file))
+            for filename in os.listdir(settings.CACHE_ROOT):
+                if filename.startswith('level_') and '_history_' in filename:
+                    os.remove(os.path.join(settings.CACHE_ROOT, filename))
 
         if not settings.HAS_REAL_CACHE:
             print(_('You have no external cache configured, so don\'t forget to restart your c3nav instance!'))

src/c3nav/mapdata/migrations/0046_remove_level_render_data.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2017-11-16 01:25
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mapdata', '0045_level_door_height'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='level',
+            name='render_data',
+        ),
+    ]

src/c3nav/mapdata/models/level.py

@@ -26,8 +26,6 @@ class Level(SpecificLocation, models.Model):
                                   related_name='levels_on_top', verbose_name=_('on top of'))
     short_label = models.SlugField(max_length=20, verbose_name=_('short label'), unique=True)
 
-    render_data = models.BinaryField(null=True)
-
     objects = LevelManager()
 
     class Meta:

src/c3nav/mapdata/render/data.py

@@ -1,4 +1,5 @@
 import operator
+import os
 import pickle
 import threading
 from collections import Counter, deque
@@ -6,6 +7,7 @@ from functools import reduce
 from itertools import chain
 
 import numpy as np
+from django.conf import settings
 from django.db import transaction
 from scipy.interpolate import NearestNDInterpolator
 from shapely import prepared
@@ -313,7 +315,7 @@ class LevelRenderData:
                 for access_restriction, areas in render_data.access_restriction_affected.items()
             }
 
-            level.render_data = pickle.dumps(render_data)
+            render_data.save(level.pk)
 
             map_history.save(MapHistory.level_filename(level.pk, 'composite'))
 
@@ -325,6 +327,10 @@ class LevelRenderData:
     cache_key = None
     cache_lock = threading.Lock()
 
+    @staticmethod
+    def _level_filename(pk):
+        return os.path.join(settings.CACHE_ROOT, 'level_%d_render_data.pickle' % pk)
+
     @classmethod
     def get(cls, level):
         with cls.cache_lock:
@@ -338,14 +344,15 @@ class LevelRenderData:
                 if result is not None:
                     return result
 
-            if isinstance(level, Level):
-                result = pickle.loads(level.render_data)
-            else:
-                result = pickle.loads(Level.objects.filter(pk=level).values_list('render_data', flat=True)[0])
+            pk = level.pk if isinstance(level, Level) else level
+            result = pickle.load(open(cls._level_filename(pk), 'rb'))
 
             cls.cached[level_pk] = result
             return result
 
+    def save(self, pk):
+        return pickle.dump(self, open(self._level_filename(pk), 'wb'))
+
 
 class Mesh:
     __slots__ = ('top', 'sides', 'bottom')