Merge branch 'main' of github.com:jennypaxian/c3nav

src/c3nav/editor/forms.py

@@ -309,13 +309,6 @@ class EditorFormBase(I18nModelFormMixin, ModelForm):
                 _('Can not add redirecting slug “%s”: it is already used elsewhere.') % slug
             )
 
-    def clean_data(self):
-        data = self.cleaned_data['data']
-        if not data.wifi:
-            raise ValidationError(_('WiFi scan data is missing.'))
-        data.wifi = [[item for item in scan if item.ssid] for scan in data.wifi]
-        return data
-
     def clean(self):
         if self.is_json:
             for name, field in self.missing_fields:
@@ -325,6 +318,16 @@ class EditorFormBase(I18nModelFormMixin, ModelForm):
             if not self.cleaned_data.get('geometry'):
                 raise ValidationError('Missing geometry.')
 
+        if 'data' in self.fields:
+            data = self.cleaned_data['data']
+            if self.cleaned_data['fill_quest']:
+                if self.cleaned_data['data'].wifi:
+                    raise ValidationError(_('Why is there WiFi scan data if this is a fill quest?'))
+            else:
+                if not self.cleaned_data['data'].wifi:
+                    raise ValidationError(_('WiFi scan data is missing.'))
+            self.cleaned_data['data'].wifi = [[item for item in scan if item.ssid] for scan in data.wifi]
+
         super().clean()
 
     def _save_m2m(self):
@@ -398,7 +401,7 @@ def create_editor_form(editor_model):
         'stroke_opacity', 'fill_color', 'fill_opacity', 'interactive', 'point_icon', 'extra_data', 'show_label',
         'show_geometry', 'show_label', 'show_geometry', 'default_geomtype', 'cluster_points', 'update_interval',
         'load_group_display', 'load_group_contribute',
-        'altitude_quest',
+        'altitude_quest', 'fill_quest',
     ]
     field_names = [field.name for field in editor_model._meta.get_fields()
                    if not field.one_to_many and not isinstance(field, ManyToManyRel)]

src/c3nav/editor/static/editor/js/editor.js

@@ -458,7 +458,10 @@ editor = {
                 collector.find('.wifi-count').text(existing_data?.wifi?.length);
                 collector.find('.ibeacon-count').text(existing_data?.ibeacon?.length);
             } else {
-                data_field.closest('form').addClass('scan-lock');
+                if (window.mobileclient) {
+                    $('[for=id_fill_quest]').hide();
+                    data_field.closest('form').addClass('scan-lock');
+                }
             }
             data_field.after(collector);
         }

src/c3nav/mapdata/management/commands/importhub.py

@@ -354,15 +354,18 @@ class Command(BaseCommand):
                           ", is now"+str([group.title for group in new_groups]), new_group_ids, old_group_ids)
 
         for import_tag, location in locations_so_far.items():
-            self.do_report(
+            if location.import_block_data:
-                prefix='hub:new_groups',
+                self.do_report(
-                obj_id=import_tag,
+                    prefix='hub:new_groups',
-                obj=import_tag,
+                    obj_id=import_tag,
-                report=Report(
+                    obj=import_tag,
-                    category="location-issue",
+                    report=Report(
-                    title="importhub: delete this",
+                        category="location-issue",
-                    description="hub wants to delete this",
+                        title="importhub: delete this",
-                    location=location,
+                        description="hub wants to delete this but it's blocked",
+                        location=location,
+                    )
                 )
-            )
+                print(f"NOTE: {location.slug} / {import_tag} should be deleted")
-            print(f"NOTE: {location.slug} / {import_tag} should be deleted")
+            else:
+                location.delete()

src/c3nav/mapdata/middleware.py

@@ -1,6 +1,7 @@
 import re
 from functools import wraps
 
+from c3nav.mapdata.utils.cache.local import per_request_cache
 from c3nav.mapdata.utils.user import get_user_data_lazy
 
 
@@ -55,3 +56,15 @@ class UserDataMiddleware:
     def __call__(self, request):
         request.user_data = get_user_data_lazy(request)
         return self.get_response(request)
+
+
+class RequestCacheMiddleware:
+    """
+    Resets the request_cache at the start of every request.
+    """
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        per_request_cache.clear()
+        return self.get_response(request)

src/c3nav/mapdata/migrations/0133_beaconmeasurement_fill_quest.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.8 on 2024-12-27 15:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mapdata', '0132_dataoverlay_update_interval_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='beaconmeasurement',
+            name='fill_quest',
+            field=models.BooleanField(default=False, verbose_name='create a quest to fill this'),
+        ),
+    ]

src/c3nav/mapdata/models/access.py

@@ -14,6 +14,7 @@ from django.utils.translation import ngettext_lazy
 
 from c3nav.mapdata.models import MapUpdate
 from c3nav.mapdata.models.base import SerializableMixin, TitledMixin
+from c3nav.mapdata.utils.cache.local import per_request_cache
 
 
 class AccessRestriction(TitledMixin, models.Model):
@@ -38,20 +39,20 @@ class AccessRestriction(TitledMixin, models.Model):
     @staticmethod
     def get_all() -> set[int]:
         cache_key = 'all_access_restrictions:%s' % MapUpdate.current_cache_key()
-        access_restriction_ids = cache.get(cache_key, None)
+        access_restriction_ids = per_request_cache.get(cache_key, None)
         if access_restriction_ids is None:
             access_restriction_ids = set(AccessRestriction.objects.values_list('pk', flat=True))
-            cache.set(cache_key, access_restriction_ids, 300)
+            per_request_cache.set(cache_key, access_restriction_ids, 300)
         return access_restriction_ids
 
     @staticmethod
     def get_all_public() -> set[int]:
         cache_key = 'public_access_restrictions:%s' % MapUpdate.current_cache_key()
-        access_restriction_ids = cache.get(cache_key, None)
+        access_restriction_ids = per_request_cache.get(cache_key, None)
         if access_restriction_ids is None:
             access_restriction_ids = set(AccessRestriction.objects.filter(public=True)
                                          .values_list('pk', flat=True))
-            cache.set(cache_key, access_restriction_ids, 300)
+            per_request_cache.set(cache_key, access_restriction_ids, 300)
         return access_restriction_ids
 
 
@@ -321,14 +322,14 @@ class AccessPermission(models.Model):
             return AccessRestriction.get_all()
 
         cache_key = cls.request_access_permission_key(request)+f':{can_grant}'
-        access_restriction_ids = cache.get(cache_key, None)
+        access_restriction_ids = per_request_cache.get(cache_key, None)
         if access_restriction_ids is None:
             permissions = cls.get_for_request_with_expire_date(request, can_grant=can_grant)
 
             access_restriction_ids = set(permissions.keys())
 
             expire_date = min((e for e in permissions.values() if e), default=timezone.now() + timedelta(seconds=120))
-            cache.set(cache_key, access_restriction_ids, min(300, (expire_date - timezone.now()).total_seconds()))
+            per_request_cache.set(cache_key, access_restriction_ids, min(300, (expire_date - timezone.now()).total_seconds()))
         return set(access_restriction_ids) | (set() if can_grant else AccessRestriction.get_all_public())
 
     @classmethod

src/c3nav/mapdata/models/geometry/space.py

@@ -447,6 +447,8 @@ class BeaconMeasurement(SpaceGeometryMixin, models.Model):
                                                     verbose_name=_('Measurement list'), 
                                                     default=BeaconMeasurementDataSchema())
 
+    fill_quest = models.BooleanField(_('create a quest to fill this'), default=False)
+
     class Meta:
         verbose_name = _('Beacon Measurement')
         verbose_name_plural = _('Beacon Measurements')

src/c3nav/mapdata/models/locations.py

@@ -23,6 +23,7 @@ from c3nav.mapdata.fields import I18nField
 from c3nav.mapdata.grid import grid
 from c3nav.mapdata.models.access import AccessRestrictionMixin
 from c3nav.mapdata.models.base import SerializableMixin, TitledMixin
+from c3nav.mapdata.utils.cache.local import per_request_cache
 from c3nav.mapdata.utils.fields import LocationById
 from c3nav.mapdata.utils.models import get_submodels
 
@@ -620,10 +621,10 @@ class Position(CustomLocationProxyMixin, models.Model):
         if not user.is_authenticated:
             return False
         cache_key = 'user_has_positions:%d' % user.pk
-        result = cache.get(cache_key, None)
+        result = per_request_cache.get(cache_key, None)
         if result is None:
             result = cls.objects.filter(owner=user).exists()
-            cache.set(cache_key, result, 600)
+            per_request_cache.set(cache_key, result, 600)
         return result
 
     def serialize_position(self, request=None):

src/c3nav/mapdata/models/update.py

@@ -16,6 +16,7 @@ from shapely.ops import unary_union
 
 from c3nav.mapdata.tasks import process_map_updates
 from c3nav.mapdata.utils.cache.changes import GeometryChangeTracker
+from c3nav.mapdata.utils.cache.local import per_request_cache
 
 
 class MapUpdate(models.Model):
@@ -48,47 +49,47 @@ class MapUpdate(models.Model):
     @classmethod
     def last_update(cls, force=False):
         if not force:
-            last_update = cache.get('mapdata:last_update', None)
+            last_update = per_request_cache.get('mapdata:last_update', None)
             if last_update is not None:
                 return last_update
         try:
             with cls.lock():
                 last_update = cls.objects.latest().to_tuple
-                cache.set('mapdata:last_update', last_update, None)
+                per_request_cache.set('mapdata:last_update', last_update, None)
         except cls.DoesNotExist:
             last_update = (0, 0)
-            cache.set('mapdata:last_update', last_update, None)
+            per_request_cache.set('mapdata:last_update', last_update, None)
         return last_update
 
     @classmethod
     def last_processed_update(cls, force=False, lock=True):
         if not force:
-            last_processed_update = cache.get('mapdata:last_processed_update', None)
+            last_processed_update = per_request_cache.get('mapdata:last_processed_update', None)
             if last_processed_update is not None:
                 return last_processed_update
         try:
             with (cls.lock() if lock else nullcontext()):
                 last_processed_update = cls.objects.filter(processed=True).latest().to_tuple
-                cache.set('mapdata:last_processed_update', last_processed_update, None)
+                per_request_cache.set('mapdata:last_processed_update', last_processed_update, None)
         except cls.DoesNotExist:
             last_processed_update = (0, 0)
-            cache.set('mapdata:last_processed_update', last_processed_update, None)
+            per_request_cache.set('mapdata:last_processed_update', last_processed_update, None)
         return last_processed_update
 
     @classmethod
     def last_processed_geometry_update(cls, force=False):
         if not force:
-            last_processed_geometry_update = cache.get('mapdata:last_processed_geometry_update', None)
+            last_processed_geometry_update = per_request_cache.get('mapdata:last_processed_geometry_update', None)
             if last_processed_geometry_update is not None:
                 return last_processed_geometry_update
         try:
             with cls.lock():
                 last_processed_geometry_update = cls.objects.filter(processed=True,
                                                                     geometries_changed=True).latest().to_tuple
-                cache.set('mapdata:last_processed_geometry_update', last_processed_geometry_update, None)
+                per_request_cache.set('mapdata:last_processed_geometry_update', last_processed_geometry_update, None)
         except cls.DoesNotExist:
             last_processed_geometry_update = (0, 0)
-            cache.set('mapdata:last_processed_geometry_update', last_processed_geometry_update, None)
+            per_request_cache.set('mapdata:last_processed_geometry_update', last_processed_geometry_update, None)
         return last_processed_geometry_update
 
     @property
@@ -239,7 +240,8 @@ class MapUpdate(models.Model):
                 LevelRenderData.rebuild(geometry_update_cache_key)
 
                 transaction.on_commit(
-                    lambda: cache.set('mapdata:last_processed_geometry_update', last_geometry_update.to_tuple, None)
+                    lambda: per_request_cache.set('mapdata:last_processed_geometry_update',
+                                              last_geometry_update.to_tuple, None)
                 )
             else:
                 logger.info('No geometries affected.')
@@ -282,7 +284,7 @@ class MapUpdate(models.Model):
 
         if new:
             transaction.on_commit(
-                lambda: cache.set('mapdata:last_update', self.to_tuple, None)
+                lambda: per_request_cache.set('mapdata:last_update', self.to_tuple, None)
             )
             if settings.HAS_CELERY and settings.AUTO_PROCESS_UPDATES:
                 transaction.on_commit(

src/c3nav/mapdata/quests/positioning.py

@@ -6,8 +6,9 @@ from django.utils.translation import gettext_lazy as _
 from shapely import Point
 from shapely.geometry import mapping
 
-from c3nav.mapdata.models.geometry.space import RangingBeacon
+from c3nav.mapdata.models.geometry.space import RangingBeacon, BeaconMeasurement
 from c3nav.mapdata.quests.base import ChangeSetModelForm, register_quest, Quest
+from c3nav.routing.schemas import BeaconMeasurementDataSchema
 
 
 class RangingBeaconAltitudeQuestForm(ChangeSetModelForm):
@@ -103,3 +104,49 @@ class RangingBeaconBSSIDsQuest(Quest):
     @classmethod
     def _qs_for_request(cls, request):
         return RangingBeacon.qs_for_request(request).filter(import_tag__startswith="noc:", wifi_bssids=[])
+
+
+class BeaconMeasurementQuestForm(ChangeSetModelForm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["data"].widget = HiddenInput()
+
+    def clean_bssids(self):
+        data = self.cleaned_data["data"]
+        if not data:
+            raise ValidationError(_("Need at least one scan."))
+        return data
+
+    class Meta:
+        model = BeaconMeasurement
+        fields = ("data", )
+
+    @property
+    def changeset_title(self):
+        return f'Beacon Measurement Quest: {self.instance.title}'
+
+
+@register_quest
+@dataclass
+class BeaconMeasurementQuest(Quest):
+    quest_type = "beacon_measurement"
+    quest_type_label = _('Wifi/BLE Positioning')
+    quest_type_icon = "wifi"
+    form_class = BeaconMeasurementQuestForm
+    obj: BeaconMeasurement
+
+    @property
+    def quest_description(self) -> list[str]:
+        return [
+            _("Please stand as close to the given location as possible. "
+              "Feel free to close this window again to double-check."),
+            _("When you're ready, please click the button below and wait for measurements to arrive."),
+        ]
+
+    @property
+    def point(self) -> Point:
+        return mapping(self.obj.geometry)
+
+    @classmethod
+    def _qs_for_request(cls, request):
+        return BeaconMeasurement.qs_for_request(request).filter(data=BeaconMeasurementDataSchema())

src/c3nav/mapdata/schemas/model_base.py

@@ -350,7 +350,7 @@ CustomLocationID = Annotated[NonEmptyStr, APIField(
 )]
 PositionID = Annotated[NonEmptyStr, APIField(
     title="position ID",
-    pattern=r"p:[A-Za-z0-9]+$",
+    pattern=r"m:[A-Za-z0-9]+$",
     description="the ID of a user-defined tracked position is made up of its secret"
 )]
 Coordinates3D = tuple[float, float, float]

src/c3nav/mapdata/utils/cache/local.py

@@ -1,8 +1,7 @@
 from collections import OrderedDict
 
 from django.core.cache import cache
-
+from django.conf import settings
-from c3nav.mapdata.models import MapUpdate
 
 
 class NoneFromCache:
@@ -42,6 +41,9 @@ class LocalCacheProxy:
             self._items.pop(next(iter(self._items.keys())))
 
     def _check_mapupdate(self):
+        # todo: would be nice to not need this… why do we need this?
+
+        from c3nav.mapdata.models import MapUpdate
         mapupdate = MapUpdate.current_cache_key()
         if self._mapupdate != mapupdate:
             self._items = OrderedDict()
@@ -52,3 +54,18 @@ class LocalCacheProxy:
         cache.set(key, value, expire)
         self._items[key] = value
         self._prune()
+
+    def clear(self):
+        self._items.clear()
+
+
+class RequestLocalCacheProxy(LocalCacheProxy):
+    """ this is a subclass without prune, to be cleared after every request """
+    def _prune(self):
+        pass
+
+    def _check_mapupdate(self):
+        pass
+
+
+per_request_cache = RequestLocalCacheProxy(maxsize=settings.CACHE_SIZE_LOCATIONS)

src/c3nav/routing/locator.py

@@ -242,7 +242,7 @@ class Locator:
         return norm
 
     def locate_range(self, scan_data: ScanData, permissions=None, orig_addr=None):
-        peer_ids = tuple(i for i in scan_data if i < len(self.xyz))
+        peer_ids = tuple(i for i, item in scan_data.items() if i < len(self.xyz) and item.distance)
 
         if len(peer_ids) < 3:
             # can't get a good result from just two beacons
@@ -262,7 +262,7 @@ class Locator:
         # create 2d array with x, y, z, distance as rows
         np_ranges = np.hstack((
             relevant_xyz,
-            np.array(tuple(scan_data[i].distance for i in peer_ids)).reshape((-1, 1)),
+            np.array(tuple(float(scan_data[i].distance) for i in peer_ids)).reshape((-1, 1)),
         ))
 
         #print(np_ranges)

src/c3nav/routing/schemas.py

@@ -115,3 +115,6 @@ class LocateIBeaconPeerSchema(BaseSchema):
 class BeaconMeasurementDataSchema(BaseSchema):
     wifi: list[list[LocateWifiPeerSchema]] = []
     ibeacon: list[list[LocateIBeaconPeerSchema]] = []
+
+    def __bool__(self):
+        return bool(self.wifi or self.ibeacon)

src/c3nav/settings.py

@@ -390,6 +390,7 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'c3nav.mapdata.middleware.RequestCacheMiddleware',
     'c3nav.mapdata.middleware.UserDataMiddleware',
     'c3nav.site.middleware.MobileclientMiddleware',
     'c3nav.control.middleware.UserPermissionsMiddleware',

src/c3nav/site/static/site/css/c3nav.scss

@@ -1999,3 +1999,21 @@ blink {
         }
     }
 }
+
+
+.ap-name-bssid-result {
+    border-radius: 4px;
+    border: 1px solid gray;
+    padding: 4px 0;
+    box-shadow: inset 0px 0px 1px gray;
+
+    thead {
+        border-bottom: 1px solid gray;
+    }
+
+    td, th {
+        padding: 0 8px;
+        border: none;
+        font-size: 0.9em;
+    }
+}

src/c3nav/site/static/site/js/c3nav.js

@@ -140,6 +140,17 @@ c3nav = {
         if (window.mobileclient) {
             $body.addClass('mobileclient');
             c3nav._set_user_location(null);
+
+            try {
+                c3nav._ap_name_mappings = JSON.parse(localStorageWrapper.getItem('c3nav.wifi-scanning.ap-names'));
+            } catch (e) {
+                // ignore
+            }
+
+            if (c3nav._ap_name_mappings === null) {
+                c3nav._ap_name_mappings = {};
+            }
+
         } else {
             document.addEventListener('visibilitychange', c3nav.on_visibility_change, false);
         }
@@ -1465,9 +1476,50 @@ c3nav = {
             .html((!no_close) ? '<button class="button-clear material-symbols" id="close-modal">clear</button>' : '')
             .append(content || '<div class="loader"></div>');
         if ($modal.find('[name=look_for_ap]').length) {
+            if (!window.mobileclient) {
+                alert('need app!')
+            }
             $modal.find('button').hide();
         }
     },
+    _ap_name_scan_result_update: function () {
+        const $modal = $('#modal');
+        const $match_ap = $modal.find('[name=look_for_ap]');
+        if ($match_ap.length) {
+            const $wifi_bssids = $('[name=wifi_bssids]');
+            const ap_name = $match_ap.val();
+            const found_bssids = {};
+            let scan_complete = false;
+            if (ap_name in c3nav._ap_name_mappings) {
+                const mappings = c3nav._ap_name_mappings[ap_name];
+                for (const mapping of mappings) {
+                    scan_complete = true;
+                    for (const bssid of mapping) {
+                        found_bssids[bssid] = (found_bssids[bssid] ?? 0) + 1;
+                        if (found_bssids[bssid] === 1) {
+                            scan_complete = false;
+                        }
+                    }
+                }
+            }
+
+            const $table = $('<table class="ap-name-bssid-result"><thead><tr><th>BSSID</th><th>count</th></tr></thead></table>')
+
+            for (const [bssid, count] of Object.entries(found_bssids)) {
+                $table.append(`<tr><td>${bssid}</td><td>${count}</td></tr>`);
+            }
+
+            $modal.find('.ap-name-bssid-result').remove();
+
+            $modal.find('form').before($table);
+
+            if (scan_complete) {
+                // todo only bssids that have count > 1
+                $wifi_bssids.val(JSON.stringify(Object.keys(found_bssids)));
+                $('#modal button[type=submit]').show();
+            }
+        }
+    },
     _modal_click: function (e) {
         if (!c3nav.modal_noclose && (e.target.id === 'modal' || e.target.id === 'close-modal')) {
             history.back();
@@ -2118,14 +2170,24 @@ c3nav = {
     _last_wifi_peers: [],
     _last_ibeacon_peers: [],
     _no_scan_count: 0,
+    _ap_name_mappings: {},
+    _enable_scan_debugging: false,
+    _scan_debugging_results: [],
     _wifi_scan_results: function (peers) {
         peers = JSON.parse(peers);
 
+        if (c3nav._enable_scan_debugging) {
+            c3nav._scan_debugging_results.push({
+                timestamp: Date.now(),
+                peers: peers,
+            });
+        }
+
         if (c3nav.ssids) {
             peers = peers.filter(peer => c3nav.ssids.includes(peer.ssid));
         }
-        let match_ap = $('[name=look_for_ap]').val(),
+
-            found_bssids = [];
+        const ap_name_mappings = {};
 
         for (const peer of peers) {
             if (peer.level !== undefined) {
@@ -2137,26 +2199,21 @@ c3nav = {
                 peer.distance_sd = peer.rtt.distance_std_dev_mm / 1000;
                 delete peer.rtt;
             }
-            if (match_ap && peer.ap_name === match_ap) {
+            if (peer.ap_name) {
-                found_bssids.push(peer.bssid);
+                let mapping = ap_name_mappings[peer.ap_name] =(ap_name_mappings[peer.ap_name] ?? new Set());
+                mapping.add(peer.bssid);
             }
         }
-        if (found_bssids.length) {
+
-            let $wifi_bssids = $('[name=wifi_bssids]'),
+        for (const [name, mapping] of Object.entries(ap_name_mappings)) {
-                val = JSON.parse($wifi_bssids.val()),
+            let mappings = c3nav._ap_name_mappings[name] = (c3nav._ap_name_mappings[name] ?? []);
-                added = 0;
+            mappings.push([...mapping]);
-            for (let bssid of found_bssids) {
-                if (!val.includes(bssid)) {
-                    val.push(bssid);
-                    added++;
-                }
-            }
-            if (added) {
-                $wifi_bssids.val(JSON.stringify(val));
-            } else {
-                $('#modal button[type=submit]').show();
-            }
         }
+
+        localStorageWrapper.setItem('c3nav.wifi-scanning.ap-names', JSON.stringify(c3nav._ap_name_mappings));
+
+        c3nav._ap_name_scan_result_update();
+
         c3nav._last_wifi_peers = peers;
         c3nav._after_scan_results();
     },