separate overlay feature data and geometries into different api endpoints so they can be cached independently

src/c3nav/editor/views/overlays.py

@@ -91,7 +91,6 @@ def overlay_feature_edit(request, level=None, overlay=None, pk=None):
     can_edit_changeset = request.changeset.can_edit(request)
 
     obj = None
-    edit_utils = DefaultEditUtils(request)
     if pk is not None:
         # Edit existing map item
         kwargs = {'pk': pk}

src/c3nav/mapdata/api/mapdata.py

@@ -27,8 +27,9 @@ from c3nav.mapdata.schemas.models import (AccessRestrictionGroupSchema, AccessRe
                                           LineObstacleSchema, LocationGroupCategorySchema, LocationGroupSchema,
                                           ObstacleSchema, POISchema, RampSchema, SourceSchema, SpaceSchema, StairSchema,
                                           DataOverlaySchema, DataOverlayFeatureSchema, LocationRedirectSchema,
-                                          WayTypeSchema,
+                                          WayTypeSchema, DataOverlayFeatureGeometrySchema,
-                                          DataOverlayFeatureUpdateSchema, DataOverlayFeatureBulkUpdateSchema)
+                                          DataOverlayFeatureUpdateSchema, DataOverlayFeatureBulkUpdateSchema,
+                                          )
 
 mapdata_api_router = APIRouter(tags=["mapdata"])
 
@@ -77,14 +78,16 @@ class MapdataEndpoint:
     model: Type[Model]
     schema: Type[BaseSchema]
     filters: Type[FilterSchema] | None = None
+    no_cache: bool = False
+    name: Optional[str] = None
 
     @property
     def model_name(self):
         return self.model._meta.model_name
 
     @property
-    def model_name_plural(self):
+    def endpoint_name(self):
-        return self.model._meta.default_related_name
+        return self.name if self.name is not None else self.model._meta.default_related_name
 
 
 @dataclass
@@ -110,7 +113,7 @@ class MapdataAPIBuilder:
             add_call_params = {}
         call_param_values = set(add_call_params.values())
         call_params = (
-            *(f"{name}={name}" for name in set(view_params.keys())-call_param_values),
+            *(f"{name}={name}" for name in set(view_params.keys()) - call_param_values),
             *(f"{name}={value}" for name, value in add_call_params.items()),
         )
         method_code = "\n".join((
@@ -139,12 +142,15 @@ class MapdataAPIBuilder:
         )
         list_func.__name__ = f"{endpoint.model_name}_list"
 
-        self.router.get(f"/{endpoint.model_name_plural}/", summary=f"{endpoint.model_name} list",
+        if not endpoint.no_cache:
+            list_func = api_etag()(list_func)
+
+        self.router.get(f"/{endpoint.endpoint_name}/", summary=f"{endpoint.model_name} list",
                         tags=[f"mapdata-{tag}"], description=schema_description(endpoint.schema),
                         response={200: list[endpoint.schema],
                                   **(validate_responses if endpoint.filters else {}),
                                   **auth_responses})(
-            api_etag()(list_func)
+            list_func
         )
 
     def add_by_id_endpoint(self, endpoint: MapdataEndpoint, tag: str):
@@ -160,7 +166,7 @@ class MapdataAPIBuilder:
         )
         list_func.__name__ = f"{endpoint.model_name}_by_id"
 
-        self.router.get(f'/{endpoint.model_name_plural}/{{{id_field}}}/', summary=f"{endpoint.model_name} by ID",
+        self.router.get(f'/{endpoint.endpoint_name}/{{{id_field}}}/', summary=f"{endpoint.model_name} by ID",
                         tags=[f"mapdata-{tag}"], description=schema_description(endpoint.schema),
                         response={200: endpoint.schema, **API404.dict(), **auth_responses})(
             api_etag()(list_func)
@@ -227,6 +233,14 @@ mapdata_endpoints: dict[str, list[MapdataEndpoint]] = {
             model=DataOverlayFeature,
             schema=DataOverlayFeatureSchema,
             filters=ByOverlayFilter,
+            no_cache=True,
+        ),
+        MapdataEndpoint(
+            model=DataOverlayFeature,
+            schema=DataOverlayFeatureGeometrySchema,
+            filters=ByOverlayFilter,
+            no_cache=True,
+            name='dataoverlayfeaturegeometries'
         ),
         MapdataEndpoint(
             model=WayType,
@@ -304,12 +318,11 @@ mapdata_endpoints: dict[str, list[MapdataEndpoint]] = {
     ],
 }
 
-
 MapdataAPIBuilder(router=mapdata_api_router).build_all_endpoints(mapdata_endpoints)
 
 
 @mapdata_api_router.post('/dataoverlayfeatures/{id}', summary="update a data overlay feature (including geometries)",
-                        response={204: None, **API404.dict(), **auth_permission_responses})
+                         response={204: None, **API404.dict(), **auth_permission_responses})
 def update_data_overlay_feature(request, id: int, parameters: DataOverlayFeatureUpdateSchema):
     """
     update the data overlay feature
@@ -317,7 +330,8 @@ def update_data_overlay_feature(request, id: int, parameters: DataOverlayFeature
 
     feature = get_object_or_404(DataOverlayFeature, id=id)
 
-    if feature.overlay.edit_access_restriction_id is None or feature.overlay.edit_access_restriction_id not in AccessPermission.get_for_request(request):
+    if feature.overlay.edit_access_restriction_id is None or feature.overlay.edit_access_restriction_id not in AccessPermission.get_for_request(
+            request):
         raise APIPermissionDenied('You are not allowed to edit this object.')
 
     updates = parameters.dict(exclude_unset=True)
@@ -329,10 +343,10 @@ def update_data_overlay_feature(request, id: int, parameters: DataOverlayFeature
 
     return 204, None
 
+
 @mapdata_api_router.post('/dataoverlayfeatures-bulk', summary="bulk-update data overlays (including geometries)",
                          response={204: None, **API404.dict(), **auth_permission_responses})
 def update_data_overlay_features_bulk(request, parameters: DataOverlayFeatureBulkUpdateSchema):
-
     permissions = AccessPermission.get_for_request(request)
 
     updates = {
@@ -343,7 +357,8 @@ def update_data_overlay_features_bulk(request, parameters: DataOverlayFeatureBul
     forbidden_object_ids = []
 
     with transaction.atomic():
-        features = DataOverlayFeature.objects.filter(id__in=updates.keys()).annotate(edit_access_restriction_id=F('overlay__edit_access_restriction_id'))
+        features = DataOverlayFeature.objects.filter(id__in=updates.keys()).annotate(
+            edit_access_restriction_id=F('overlay__edit_access_restriction_id'))
 
         for feature in features:
             if feature.edit_access_restriction_id is None or feature.edit_access_restriction_id not in permissions:

src/c3nav/mapdata/migrations/0131_dataoverlay_update_interval_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.3 on 2024-12-26 19:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mapdata', '0130_dataoverlay_edit_access_restriction'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataoverlay',
+            name='update_interval',
+            field=models.PositiveIntegerField(blank=True, help_text='in seconds', null=True, verbose_name='frontend update interval'),
+        ),
+        migrations.AlterField(
+            model_name='dataoverlay',
+            name='fill_opacity',
+            field=models.FloatField(blank=True, null=True, verbose_name='default fill opacity'),
+        ),
+    ]

src/c3nav/mapdata/models/overlay.py

@@ -24,16 +24,21 @@ class DataOverlay(TitledMixin, AccessRestrictionMixin, models.Model):
     stroke_width = models.FloatField(blank=True, null=True, verbose_name=_('default stroke width'))
     stroke_opacity = models.FloatField(blank=True, null=True, verbose_name=_('stroke opacity'))
     fill_color = models.CharField(max_length=255, blank=True, null=True, verbose_name=_('default fill color'))
-    fill_opacity = models.FloatField(blank=True, null=True, verbose_name=_('fill opacity'))
+    fill_opacity = models.FloatField(blank=True, null=True, verbose_name=_('default fill opacity'))
 
     cluster_points = models.BooleanField(default=False, verbose_name=_('cluster points together when zoomed out'))
 
-    default_geomtype = models.CharField(max_length=255, blank=True, null=True, choices=GeometryType, verbose_name=_('default geometry type'))
+    default_geomtype = models.CharField(max_length=255, blank=True, null=True, choices=GeometryType,
+                                        verbose_name=_('default geometry type'))
 
     pull_url = models.URLField(blank=True, null=True, verbose_name=_('pull URL'))
     pull_headers: dict[str, str] = SchemaField(schema=dict[str, str], null=True,
                                                verbose_name=_('headers for pull http request (JSON object)'))
     pull_interval = models.DurationField(blank=True, null=True, verbose_name=_('pull interval'))
+
+    update_interval = models.PositiveIntegerField(blank=True, null=True, verbose_name=_('frontend update interval'),
+                                                  help_text=_('in seconds'))
+
     edit_access_restriction = models.ForeignKey(AccessRestriction, null=True, blank=True,
                                                 related_name='edit_access_restrictions',
                                                 verbose_name=_('Editor Access Restriction'),
@@ -46,7 +51,8 @@ class DataOverlay(TitledMixin, AccessRestrictionMixin, models.Model):
 
 
 class DataOverlayFeature(TitledMixin, LevelGeometryMixin, models.Model):
-    overlay = models.ForeignKey('mapdata.DataOverlay', on_delete=models.CASCADE, verbose_name=_('Overlay'), related_name='features')
+    overlay = models.ForeignKey('mapdata.DataOverlay', on_delete=models.CASCADE, verbose_name=_('Overlay'),
+                                related_name='features')
     geometry = GeometryField()
     # level = models.ForeignKey('mapdata.Level', on_delete=models.CASCADE, verbose_name=_('level'), related_name='data_overlay_features')
     external_url = models.URLField(blank=True, null=True, verbose_name=_('external URL'))
@@ -62,9 +68,10 @@ class DataOverlayFeature(TitledMixin, LevelGeometryMixin, models.Model):
     point_icon = models.CharField(max_length=255, blank=True, null=True, verbose_name=_('point icon'),
                                   help_text=_(
                                       'use this material icon to display points, instead of drawing a small circle (only makes sense if the geometry is a point)'))
-    extra_data: Optional[dict[str, str|int|bool]] = SchemaField(schema=dict[str, str|int|bool], blank=True, null=True,
+    extra_data: Optional[dict[str, str | int | bool]] = SchemaField(schema=dict[str, str | int | bool], blank=True,
-                                                                default=None,
+                                                                    null=True,
-                                                                verbose_name=_('extra data (JSON object)'))
+                                                                    default=None,
+                                                                    verbose_name=_('extra data (JSON object)'))
 
     def to_geojson(self, instance=None) -> dict:
         result = {

src/c3nav/mapdata/schemas/models.py

@@ -376,14 +376,13 @@ class DataOverlaySchema(TitledSchema, DjangoModelSchema):
     fill_color: Optional[str]
     fill_opacity: Optional[float]
     cluster_points: bool
+    update_interval: Optional[PositiveInt]
 
 
-
+class DataOverlayFeatureSchema(TitledSchema, DjangoModelSchema):
-class DataOverlayFeatureSchema(TitledSchema, WithGeometrySchema, DjangoModelSchema):
     """
     A feature (any kind of geometry) to be displayed as part of a data overlay.
     """
-    geometry: AnyGeometrySchema
     level_id: PositiveInt
     stroke_color: Optional[str]
     stroke_width: Optional[float]
@@ -397,6 +396,13 @@ class DataOverlayFeatureSchema(TitledSchema, WithGeometrySchema, DjangoModelSche
     external_url: Optional[str]
     extra_data: Optional[dict[str, str | int | float]]
 
+
+class DataOverlayFeatureGeometrySchema(WithGeometrySchema, DjangoModelSchema):
+    """
+    A feature (any kind of geometry) to be displayed as part of a data overlay.
+    """
+    geometry: AnyGeometrySchema
+
 class DataOverlayFeatureUpdateSchema(BaseSchema):
     """
     An update to a data overlay feature.

src/c3nav/site/static/site/css/c3nav.scss

@@ -1975,29 +1975,25 @@ blink {
 }
 
 .marker-cluster {
-	background-color: color-mix(in srgb, transparent, var(--color-primary) 60%);
+    span {
-    background-clip: padding-box;
+        display: inline-block;
-	border-radius: 20px;
-
-    div {
-        background-color: white;
-        color: var(--color-primary);
         width: 30px;
         height: 30px;
-        margin-left: 5px;
+        line-height: 30px;
-        margin-top: 5px;
-
         text-align: center;
-        border-radius: 15px;
+        background-color: white;
-        font-size: 12px;
+        color: var(--cluster-marker-color);
+        font-weight: bold;
+        border-radius: 100%;
 
-        span {
+        box-shadow: 0 0 0 5px color-mix(in srgb, transparent, var(--cluster-marker-color) 60%);
-            line-height: 30px;
+
-	        font-weight: bold;
+        transition: color, background-color 150ms ease-in-out;
-        }
+
+        cursor: pointer;
 
         &:hover {
-            background-color: var(--color-primary);
+            background-color: var(--cluster-marker-color);
             color: white;
         }
     }

src/c3nav/site/static/site/js/c3nav.js

@@ -47,6 +47,17 @@
     };
 }());
 
+function makeClusterIconCreate(color) {
+    return function(cluster) {
+        const childCount = cluster.getChildCount();
+        return new L.DivIcon({
+            html: `<div style="--cluster-marker-color: ${color};"><span>${childCount}</span></div>`,
+            className: 'marker-cluster',
+            iconSize: new L.Point(30, 30)
+        });
+    }
+}
+
 /**
  * a wrapper for localStorage, catching possible exception when accessing or setting data.
  * working silently if there are errors apart from a console log message when setting an item.
@@ -1586,6 +1597,7 @@ c3nav = {
                     color: 'var(--color-primary)',
                 },
                 showCoverageOnHover: false,
+                iconCreateFunction: makeClusterIconCreate('var(--color-primary)'),
             }).addTo(layerGroup);
         }
         c3nav._levelControl.finalize();
@@ -3130,6 +3142,7 @@ L.SquareGridLayer = L.Layer.extend({
 
 class DataOverlay {
     levels = null;
+    feature_geometries = {};
 
     constructor(options) {
         this.id = options.id;
@@ -3141,19 +3154,66 @@ class DataOverlay {
         this.default_stroke_opacity = options.stroke_opacity;
         this.default_fill_color = options.fill_color;
         this.default_fill_opacity = options.fill_opacity;
+        this.update_interval = options.update_interval === null ? null : options.update_interval * 1000;
     }
 
     async create() {
-        const features = await c3nav_api.get(`mapdata/dataoverlayfeatures/?overlay=${this.id}`);
+        const [features, feature_geometries] = await Promise.all([
+            c3nav_api.get(`mapdata/dataoverlayfeatures/?overlay=${this.id}`),
+            c3nav_api.get(`mapdata/dataoverlayfeaturegeometries/?overlay=${this.id}`)
+        ]);
+
+        this.feature_geometries = Object.fromEntries(feature_geometries.map(f => [f.id, f.geometry]));
+
+        this.update_features(features);
+
+        if (this.update_interval !== null) {
+            window.setTimeout(() => {
+                this.fetch_features()
+                    .catch(err => console.error(err))
+            }, this.update_interval);
+        }
+    }
+
+    async fetch_features() {
+        const features= await c3nav_api.get(`mapdata/dataoverlayfeatures/?overlay=${this.id}`);
+
+        this.update_features(features);
+
+        if (this.update_interval !== null) {
+            window.setTimeout(() => {
+                this.fetch_features()
+                    .catch(err => console.error(err))
+            }, this.update_interval);
+        }
+    }
+
+    update_features (features) {
+        if (this.levels === null) {
+            this.levels = {};
+        }
+
+        for (let id in this.levels) {
+            this.levels[id].clearLayers();
+        }
 
-        const levels = {};
         for (const feature of features) {
+            const geometry = this.feature_geometries[feature.id]
             const level_id = feature.level_id;
-            if (!(level_id in levels)) {
+            if (!(level_id in this.levels)) {
                 if (this.cluster_points) {
-                    levels[level_id] = L.markerClusterGroup();
+                    this.levels[level_id] = L.markerClusterGroup({
+                        spiderLegPolylineOptions: {
+                            color: this.default_stroke_color ?? 'var(--color-map-overlay)',
+                        },
+                        polygonOptions: {
+                            color: this.default_stroke_color ?? 'var(--color-map-overlay)',
+                            fillColor: this.default_fill_color ?? 'var(--color-map-overlay)',
+                        },
+                        iconCreateFunction: makeClusterIconCreate(this.default_fill_color ?? 'var(--color-map-overlay)'),
+                    });
                 } else {
-                    levels[level_id] = L.layerGroup();
+                    this.levels[level_id] = L.layerGroup();
                 }
             }
             const style = {
@@ -3163,7 +3223,7 @@ class DataOverlay {
                 'fillColor': feature.fill_color ?? this.default_fill_color ?? 'var(--color-map-overlay)',
                 'fillOpacity': feature.fill_opacity ?? this.default_fill_opacity ?? 0.2,
             };
-            const layer = L.geoJson(feature.geometry, {
+            const layer = L.geoJson(geometry, {
                 style,
                 interactive: feature.interactive,
                 pointToLayer: (geom, latlng) => {
@@ -3176,22 +3236,6 @@ class DataOverlay {
                             iconAnchor: [12, 12],
                         })
                     });
-                    if (feature.point_icon !== null) {
-                        return L.marker(latlng, {
-                            title: feature.title,
-                            icon: L.divIcon({
-                                className: 'symbol-icon ' + (feature.interactive ? 'symbol-icon-interactive' : ''),
-                                html: `<span style="--icon-color: ${style.color}">${feature.point_icon}</span>`,
-                                iconSize: [24, 24],
-                                iconAnchor: [12, 12],
-                            })
-                        });
-                    } else {
-                        return L.circleMarker(latlng, {
-                            title: feature.title,
-                            ...style
-                        });
-                    }
                 },
                 onEachFeature: (f, layer) => {
                     if (feature.interactive) {
@@ -3216,10 +3260,8 @@ class DataOverlay {
                 }
             });
 
-            levels[level_id].addLayer(layer);
+            this.levels[level_id].addLayer(layer);
         }
-
-        this.levels = levels;
     }
 
     async enable(levels) {