add options and postpone imports to massively reduce memory usage

src/c3nav/control/forms.py

@@ -10,8 +10,7 @@ from typing import Sequence
 
 from django.contrib.auth.models import User
 from django.db.models import Prefetch
-from django.forms import (ChoiceField, Form, IntegerField, ModelForm, ModelMultipleChoiceField, MultipleChoiceField,
+from django.forms import ChoiceField, Form, IntegerField, ModelForm, Select
-                          Select)
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.utils.translation import ngettext_lazy
@@ -22,9 +21,6 @@ from c3nav.mapdata.forms import I18nModelFormMixin
 from c3nav.mapdata.models import MapUpdate, Space
 from c3nav.mapdata.models.access import (AccessPermission, AccessPermissionToken, AccessPermissionTokenItem,
                                          AccessRestriction, AccessRestrictionGroup)
-from c3nav.mesh.messages import MeshMessageType
-from c3nav.mesh.models import MeshNode
-from c3nav.mesh.utils import group_msg_type_choices
 from c3nav.site.models import Announcement
 
 
@@ -341,16 +337,3 @@ class MapUpdateForm(ModelForm):
     class Meta:
         model = MapUpdate
         fields = ('geometries_changed', )
-
-
-class MeshMessageFilterForm(Form):
-    message_types = MultipleChoiceField(
-        choices=group_msg_type_choices(list(MeshMessageType)),
-        required=False,
-        label=_('message types'),
-    )
-    src_nodes = ModelMultipleChoiceField(
-        queryset=MeshNode.objects.all(),
-        required=False,
-        label=_('nodes'),
-    )

src/c3nav/mapdata/models/geometry/level.py

@@ -11,7 +11,6 @@ from django.urls import reverse
 from django.utils.functional import cached_property
 from django.utils.text import format_lazy
 from django.utils.translation import gettext_lazy as _
-from scipy.sparse.csgraph._shortest_path import dijkstra
 from shapely import prepared
 from shapely.affinity import scale
 from shapely.geometry import JOIN_STYLE, LineString, MultiPolygon
@@ -352,6 +351,7 @@ class AltitudeArea(LevelGeometryMixin, models.Model):
 
         repeat = True
 
+        from scipy.sparse.csgraph._shortest_path import dijkstra
         while repeat:
             repeat = False
             # noinspection PyTupleAssignmentBalance

src/c3nav/mapdata/render/geometry/hybrid.py

@@ -14,7 +14,6 @@ from shapely.ops import unary_union
 from c3nav.mapdata.render.geometry.mesh import Mesh
 from c3nav.mapdata.utils.geometry import assert_multipolygon
 from c3nav.mapdata.utils.mesh import triangulate_polygon
-from c3nav.mapdata.utils.mpl import shapely_to_mpl
 
 
 def hybrid_union(geoms):
@@ -57,6 +56,7 @@ class HybridGeometry:
         """
         if isinstance(geom, (LineString, MultiLineString)):
             return HybridGeometry(geom, ())
+        from c3nav.mapdata.utils.mpl import shapely_to_mpl  # moved in here to save memory
         faces = tuple(
             set(np.argwhere(shapely_to_mpl(subgeom).contains_points(face_centers)).flatten())
             for subgeom in assert_multipolygon(geom)

src/c3nav/mapdata/render/geometry/level.py

@@ -4,7 +4,6 @@ from functools import reduce
 from itertools import chain
 
 import numpy as np
-from scipy.interpolate import NearestNDInterpolator
 from shapely import prepared
 from shapely.geometry import GeometryCollection
 from shapely.ops import unary_union
@@ -309,6 +308,8 @@ class LevelGeometries:
             vertex_values[i_vertices] = value_func(item, i_vertices)
             vertex_value_mask[i_vertices] = True
 
+        from scipy.interpolate import NearestNDInterpolator  # moved in here to save memory
+
         if np.any(vertex_value_mask) and not np.all(vertex_value_mask):
             interpolate = NearestNDInterpolator(self.vertices[vertex_value_mask],
                                                 vertex_values[vertex_value_mask])

src/c3nav/mapdata/render/renderdata.py

@@ -7,7 +7,6 @@ from typing import Optional
 
 import numpy as np
 from django.conf import settings
-from scipy.interpolate import NearestNDInterpolator
 from shapely import Geometry, MultiPolygon, prepared
 from shapely.geometry import GeometryCollection
 from shapely.ops import unary_union
@@ -68,6 +67,8 @@ class LevelRenderData:
         # todo: we should check that levels on top come before their levels as they should
 
         themes = [None, *Theme.objects.values_list('pk', flat=True)]
+        from scipy.interpolate import NearestNDInterpolator  # moved in here to save memory
+
         from c3nav.mapdata.render.theme import ColorManager
 
         for theme in themes:

src/c3nav/mapdata/utils/geometry.py

@@ -3,11 +3,8 @@ from collections import deque, namedtuple
 from itertools import chain
 from typing import List, Sequence, Union
 
-import matplotlib.pyplot as plt
 from django.core import checks
 from django.utils.functional import cached_property
-from matplotlib.patches import PathPatch
-from matplotlib.path import Path
 from shapely import prepared, speedups
 from shapely.geometry import GeometryCollection, LinearRing, LineString, MultiLineString, MultiPolygon, Point, Polygon
 from shapely.geometry import mapping as shapely_mapping
@@ -120,6 +117,11 @@ def good_representative_point(geometry):
 
 
 def plot_geometry(geom, title=None, bounds=None):
+    # these imports live here so they are only imported when needed
+    import matplotlib.pyplot as plt
+    from matplotlib.patches import PathPatch
+    from matplotlib.path import Path
+
     fig = plt.figure()
     axes = fig.add_subplot(111)
     if bounds is None:

src/c3nav/mesh/forms.py

@@ -10,7 +10,7 @@ from asgiref.sync import async_to_sync
 from django import forms
 from django.core.exceptions import ValidationError
 from django.db import transaction
-from django.forms import BooleanField, ChoiceField, Form
+from django.forms import BooleanField, ChoiceField, Form, ModelMultipleChoiceField, MultipleChoiceField
 from django.http import Http404
 from django.utils.translation import gettext_lazy as _
 
@@ -18,7 +18,7 @@ from c3nav.mesh.dataformats import BoardConfig, BoardType, LedType, SerialLedTyp
 from c3nav.mesh.messages import MESH_BROADCAST_ADDRESS, MESH_ROOT_ADDRESS, MeshMessage, MeshMessageType
 from c3nav.mesh.models import (FirmwareBuild, HardwareDescription, MeshNode, OTARecipientStatus, OTAUpdate,
                                OTAUpdateRecipient)
-from c3nav.mesh.utils import MESH_ALL_OTA_GROUP
+from c3nav.mesh.utils import MESH_ALL_OTA_GROUP, group_msg_type_choices
 
 
 class MeshMessageForm(forms.Form):
@@ -400,3 +400,16 @@ class OTACreateForm(Form):
             "addresses": addresses,
         })
         return updates
+
+
+class MeshMessageFilterForm(Form):
+    message_types = MultipleChoiceField(
+        choices=group_msg_type_choices(list(MeshMessageType)),
+        required=False,
+        label=_('message types'),
+    )
+    src_nodes = ModelMultipleChoiceField(
+        queryset=MeshNode.objects.all(),
+        required=False,
+        label=_('nodes'),
+    )

src/c3nav/mesh/views/messages.py

@@ -8,8 +8,7 @@ from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django.views.generic import FormView, ListView, TemplateView
 
-from c3nav.control.forms import MeshMessageFilterForm
+from c3nav.mesh.forms import MeshMessageFilterForm, MeshMessageForm
-from c3nav.mesh.forms import MeshMessageForm
 from c3nav.mesh.messages import MeshMessage, MeshMessageType
 from c3nav.mesh.models import MeshNode, NodeMessage
 from c3nav.mesh.utils import get_node_names, group_msg_type_choices

src/c3nav/routing/locator.py

@@ -1,14 +1,13 @@
 import operator
 import pickle
 from dataclasses import dataclass, field
-from functools import reduce
+from functools import cached_property, reduce
 from pprint import pprint
 from typing import Optional, Self, Sequence, TypeAlias
 
 import numpy as np
 import scipy
 from django.conf import settings
-from scipy.optimize import least_squares
 
 from c3nav.mapdata.models import MapUpdate, Space
 from c3nav.mapdata.models.geometry.space import RangingBeacon
@@ -198,6 +197,12 @@ class Locator:
 
         return best_location
 
+    @cached_property
+    def least_squares_func(self):
+        # this is effectively a lazy import to save memory… todo: do we need that?
+        from scipy.optimize import least_squares
+        return least_squares
+
     def locate_range(self, scan_data: ScanData, permissions=None, orig_addr=None):
         pprint(scan_data)
 
@@ -248,7 +253,7 @@ class Locator:
         initial_guess = np.average(np_ranges[:, :dimensions], axis=0)
 
         # here the magic happens
-        results = least_squares(
+        results = self.least_squares_func(
             fun=cost_func,
             # jac="3-point",
             loss="linear",

src/c3nav/routing/router.py

@@ -10,7 +10,6 @@ import numpy as np
 from django.conf import settings
 from django.core.cache import cache
 from django.utils.functional import cached_property
-from scipy.sparse.csgraph._shortest_path import shortest_path
 from shapely import prepared
 from shapely.geometry import LineString, Point
 from shapely.ops import unary_union
@@ -399,6 +398,12 @@ class Router:
         return CustomLocationDescription(space=space, altitude=altitude,
                                          areas=areas, near_area=near_area, near_poi=near_poi, nearby=nearby)
 
+    @cached_property
+    def shortest_path_func(self):
+        # this is effectively a lazy import to save memory… todo: do we need that?
+        from scipy.sparse.csgraph._shortest_path import shortest_path
+        return shortest_path
+
     def shortest_path(self, restrictions, options):
         options_key = options.serialize_string()
         cache_key = 'router:shortest_path:%s:%s:%s' % (MapUpdate.current_processed_cache_key(),
@@ -469,7 +474,7 @@ class Router:
             graph[:, tuple(restrictions.additional_nodes)] = np.inf
         graph[tuple(restrictions.edges.transpose().tolist())] = np.inf
 
-        distances, predecessors = shortest_path(graph, directed=True, return_predecessors=True)
+        distances, predecessors = self.shortest_path_func(graph, directed=True, return_predecessors=True)
         cache.set(cache_key, (distances.astype(np.float64).tobytes(),
                               predecessors.astype(np.int32).tobytes()), 600)
         return distances, predecessors

src/c3nav/settings.py

@@ -151,6 +151,10 @@ if not SECRET_MESH_KEY:
 debug_fallback = "runserver" in sys.argv
 DEBUG = config.getboolean('django', 'debug', fallback=debug_fallback, env='C3NAV_DEBUG')
 
+ENABLE_MESH = config.getboolean('django', 'enable_mesh', fallback=True, env='ENABLE_MESH')
+SERVE_API = config.getboolean('django', 'serve_api', fallback=True, env='SERVE_API')
+SERVE_ANYTHING = config.getboolean('django', 'serve_anything', fallback=True, env='SERVE_ANYTHING')
+
 RENDER_SCALE = config.getfloat('c3nav', 'render_scale', fallback=20.0)
 IMAGE_RENDERER = config.get('c3nav', 'image_renderer', fallback='svg')
 SVG_RENDERER = config.get('c3nav', 'svg_renderer', fallback='rsvg-convert')
@@ -324,7 +328,7 @@ TILE_ACCESS_COOKIE_SAMESITE = 'none' if SESSION_COOKIE_SECURE else 'lax'
 # Application definition
 
 INSTALLED_APPS = [
-    "daphne",
+    *(["daphne"] if DEBUG else []),
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -334,13 +338,13 @@ INSTALLED_APPS = [
     'channels',
     'compressor',
     'bootstrap3',
-    'ninja',
+    *(["ninja"] if SERVE_API else []),
     'c3nav.api',
     'c3nav.mapdata',
     'c3nav.routing',
     'c3nav.site',
     'c3nav.control',
-    'c3nav.mesh',
+    *(["c3nav.mesh"] if ENABLE_MESH else []),
     'c3nav.editor',
 ]
 
@@ -357,7 +361,7 @@ MIDDLEWARE = [
     'c3nav.mapdata.middleware.UserDataMiddleware',
     'c3nav.site.middleware.MobileclientMiddleware',
     'c3nav.control.middleware.UserPermissionsMiddleware',
-    'c3nav.api.middleware.JsonRequestBodyMiddleware',
+    #'c3nav.api.middleware.JsonRequestBodyMiddleware',  # might still be needed in editor
 ]
 
 with suppress(ImportError):

src/c3nav/urls.py

@@ -6,29 +6,44 @@ from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
 
-import c3nav.api.urls
+urlpatterns = []
-import c3nav.control.urls
+websocket_urlpatterns = []
-import c3nav.editor.urls
-import c3nav.mapdata.urls
-import c3nav.mesh.urls
-import c3nav.site.urls
 
-urlpatterns = [
+if settings.SERVE_ANYTHING:
-    path('editor/', include(c3nav.editor.urls)),
+    import c3nav.control.urls
-    path('api/', include(c3nav.api.urls)),
+    import c3nav.editor.urls
-    path('map/', include(c3nav.mapdata.urls)),
+    import c3nav.mapdata.urls
-    path('admin/', admin.site.urls),
+    import c3nav.site.urls
-    path('control/', include(c3nav.control.urls)),
+    urlpatterns += [
-    path('mesh/', include(c3nav.mesh.urls)),
+        path('editor/', include(c3nav.editor.urls)),
-    path('locales/', include('django.conf.urls.i18n')),
+        path('map/', include(c3nav.mapdata.urls)),
-    path('', include(c3nav.site.urls)),
+        path('admin/', admin.site.urls),
-] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+        path('control/', include(c3nav.control.urls)),
+    ]
 
-websocket_urlpatterns = [
+    if settings.SERVE_API:
-    path('mesh/', URLRouter(c3nav.mesh.urls.websocket_urlpatterns)),
+        import c3nav.api.urls
-]
+        urlpatterns += [
+            path('api/', include(c3nav.api.urls)),
+        ]
 
-if settings.DEBUG:
+    if settings.ENABLE_MESH:
-    with suppress(ImportError):
+        import c3nav.mesh.urls
-        import debug_toolbar
+        urlpatterns += [
-        urlpatterns.insert(0, path('__debug__/', include(debug_toolbar.urls)))
+            path('mesh/', include(c3nav.mesh.urls)),
+        ]
+
+    urlpatterns += [
+        path('locales/', include('django.conf.urls.i18n')),
+        path('', include(c3nav.site.urls)),
+    ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+
+    if settings.ENABLE_MESH:
+        websocket_urlpatterns += [
+            path('mesh/', URLRouter(c3nav.mesh.urls.websocket_urlpatterns)),
+        ]
+
+    if settings.DEBUG:
+        with suppress(ImportError):
+            import debug_toolbar
+            urlpatterns.insert(0, path('__debug__/', include(debug_toolbar.urls)))