importhub now with automatic reports doing

src/c3nav/mapdata/management/commands/importhub.py

@@ -1,3 +1,5 @@
+import hashlib
+import json
 from uuid import UUID
 
 import requests
@@ -11,6 +13,7 @@ from shapely.geometry import shape
 from c3nav.api.utils import NonEmptyStr
 from c3nav.mapdata.models import Area, Space, LocationGroup, LocationSlug, MapUpdate
 from c3nav.mapdata.models.geometry.space import POI
+from c3nav.mapdata.models.report import Report
 from c3nav.mapdata.utils.cache.changes import changed_geometries
 
 
@@ -45,6 +48,15 @@ class Command(BaseCommand):
             self.do_import(r.json())
             MapUpdate.objects.create(type='importhub')
 
+    def do_report(self, prefix: str, obj_id: str, obj, report: Report):
+        import_prefix = f"{prefix}:{obj_id}:"
+        import_tag = import_prefix+hashlib.md5(str(obj).encode()).hexdigest()
+        Report.objects.filter(import_tag__startswith=import_prefix, open=True).exclude(import_tag=import_tag).delete()
+        if not Report.objects.filter(import_tag=import_tag).exists():
+            report.import_tag = import_tag
+            report.save()
+            report.notify_reviewers()
+
     def do_import(self, data):
         items: list[HubImportItem] = [HubImportItem.model_validate(item) for item in data]
         items_by_id = {item.id: item for item in items}
@@ -118,8 +130,31 @@ class Command(BaseCommand):
                 if not isinstance(result, target_type):
                     # need to change from POI to Area or inverted
                     if result.import_block_geom:
-                        print(f"ERROR: {item.slug} / {item.id} needs to be switched to {target_type} but is blocked")
+                        self.do_report(
+                            prefix='hub:switch_type',
+                            obj_id=str(item.id),
+                            report=Report(
+                                category="location-issue",
+                                title=f"importhub: change geometry type for {result.title}, is blocked",
+                                description=f"the object has a wrong geometry type and needs to be switched to "
+                                            f"{target_type} but the geometry is blocked",
+                                location=result,
+                            )
+                        )
+                        print(f"REPORT: {item.slug} / {item.id} needs to be switched to {target_type} but is blocked")
                         continue
+                    self.do_report(
+                        prefix='hub:switch_type',
+                        obj_id=str(item.id),
+                        obj=item,
+                        report=Report(
+                            category="location-issue",
+                            title=f"importhub: change geometry type for {result.title}, not implemented",
+                            description=f"the object has a wrong geometry type and needs to be switched to "
+                                        f"{target_type} but this is not implemented yet",
+                            location=result,
+                        )
+                    )
                     print(f"ERROR: {item.slug} / {item.id} needs to be switched to {target_type} but not implemented")
                     continue
 
@@ -140,14 +175,18 @@ class Command(BaseCommand):
                     import_tag=import_tag,
                 )
 
+            geometry_needs_change = []
+
             if result.space_id != new_space.pk:
                 if result.import_block_geom:
+                    geometry_needs_change.append(f"change to space {new_space.title}")
                     print(f"NOTE: {item.slug} / {item.id} space has changed but is blocked")
                 else:
                     result.space_id = new_space.pk
 
             if result.geometry != new_geometry or True:
                 if result.import_block_geom:
+                    geometry_needs_change.append(f"change geometry")
                     print(f"NOTE: {item.slug} / {item.id} geometry has changed but is blocked")
                 else:
                     result.geometry = new_geometry
@@ -156,16 +195,60 @@ class Command(BaseCommand):
                 new_main_point = Point(item.location) if item.location else None
                 if result.main_point != new_main_point:
                     if result.import_block_geom:
+                        geometry_needs_change.append(f"change main point")
                         print(f"NOTE: {item.slug} / {item.id} main point has changed but is blocked")
                     else:
                         result.main_point = new_main_point
 
+            if geometry_needs_change:
+                self.do_report(
+                    prefix='hub:change_geometry',
+                    obj_id=str(item.id),
+                    obj=item,
+                    report=Report(
+                        category="location-issue",
+                        title=f"importhub: geometry is blocked but needs changing",
+                        description=f"changes needed: "+','.join(geometry_needs_change),
+                        location=result,
+                    )
+                )
+
+            data_needs_change = []
+
             if item.slug != result.slug:
                 if result.import_block_data:
                     print(f"NOTE: {item.slug} / {item.id} slug has changed but is blocked")
+                    data_needs_change.append(f"change slug to {item.slug}")
                 else:
-                    if LocationSlug.objects.filter(slug=item.slug):
+                    slug_occupied = LocationSlug.objects.filter(slug=item.slug).first()
+                    if slug_occupied:
                         print(f"ERROR: {item.slug} / {item.id} slug {item.slug!r} is already occupied")
+                        if is_new:
+                            self.do_report(
+                                prefix='hub:new_slug_occupied',
+                                obj_id=str(item.id),
+                                obj=item,
+                                report=Report(
+                                    category="location-issue",
+                                    title=f"importhub: want to import item with this slug ({item.slug}), occupied",
+                                    description=f"object to add {item.id}, for slug '{item.slug}' has name {item.name} "
+                                                f"and url {item.public_url} and should be in space {new_space.title}",
+                                    location=slug_occupied,
+                                )
+                            )
+                        else:
+                            self.do_report(
+                                prefix='hub:new_slug_occupied',
+                                obj_id=str(item.id),
+                                obj=item,
+                                report=Report(
+                                    category="location-issue",
+                                    title=f"importhub: want change slug to {item.slug} but it's occupied",
+                                    description=f"object to add {item.id} for slug '{item.slug}' has name {item.name} "
+                                                f"and url {item.public_url}",
+                                    location=result,
+                                )
+                            )
                         continue
                     else:
                         result.slug = item.slug
@@ -174,15 +257,30 @@ class Command(BaseCommand):
             if new_titles != result.titles:
                 if result.import_block_data:
                     print(f"NOTE: {item.slug} / {item.id} name has changed but is blocked")
+                    data_needs_change.append(f"change name to {item.name}")
                 else:
                     result.titles = new_titles
 
             if item.public_url != result.external_url:
                 if result.import_block_data:
                     print(f"NOTE: {item.slug} / {item.id} external url has changed but is blocked")
+                    data_needs_change.append(f"change external_url to {item.public_url}")
                 else:
                     result.external_url = item.public_url
 
+            if data_needs_change:
+                self.do_report(
+                    prefix='hub:change_data',
+                    obj_id=str(item.id),
+                    obj=item,
+                    report=Report(
+                        category="location-issue",
+                        title=f"importhub: data is blocked but needs changing",
+                        description=f"changes needed: "+','.join(data_needs_change),
+                        location=result,
+                    )
+                )
+
             # time to check the groups
             new_group_ids = set(group.id for group in new_groups)
             if is_new:
@@ -192,6 +290,17 @@ class Command(BaseCommand):
             else:
                 if not new_group_ids:
                     print(f"SKIPPING: {item.slug} / {item.id} no longer has any group IDs, {hub_types}")
+                    self.do_report(
+                        prefix='hub:new_groups',
+                        obj_id=str(item.id),
+                        obj=item,
+                        report=Report(
+                            category="location-issue",
+                            title=f"importhub: location no longer has any valid group ids",
+                            description=f"from the hub we would remove all groups, this seems wrong",
+                            location=result,
+                        )
+                    )
                     continue
 
             result.save()
@@ -200,10 +309,33 @@ class Command(BaseCommand):
             else:
                 old_group_ids = set(group.pk for group in result.groups.all())
                 if new_group_ids != old_group_ids:
+                    self.do_report(
+                        prefix='hub:new_groups',
+                        obj_id=str(item.id),
+                        obj=item,
+                        report=Report(
+                            category="location-issue",
+                            title=f"importhub: new groups",
+                            description=(f"hub wants new groups for this, groups are now: " +
+                                         str([group.title for group in new_groups])),
+                            location=result,
+                        )
+                    )
                     print(f"NOTE: {item.slug} / {item.id} groups have changed, was " +
                           str([group.title for group in result.groups.all()]) +
                           ", is now"+str([group.title for group in new_groups]), new_group_ids, old_group_ids)
 
         for import_tag, location in locations_so_far.items():
+            self.do_report(
+                prefix='hub:new_groups',
+                obj_id=import_tag,
+                obj=import_tag,
+                report=Report(
+                    category="location-issue",
+                    title=f"importhub: delete this",
+                    description=f"hub wants to delete this",
+                    location=location,
+                )
+            )
             print(f"NOTE: {location.slug} / {import_tag} should be deleted")
 

src/c3nav/mapdata/migrations/0098_report_import_tag.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2023-12-25 21:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mapdata', '0097_longer_import_tag'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='report',
+            name='import_tag',
+            field=models.CharField(blank=True, max_length=256, null=True, verbose_name='import tag'),
+        ),
+    ]

src/c3nav/mapdata/models/report.py

@@ -54,6 +54,8 @@ class Report(models.Model):
                                             help_text=_('select all groups that apply, if any'), related_name='+')
     secret = models.CharField(_('secret'), max_length=32, default=get_report_secret)
 
+    import_tag = models.CharField(_('import tag'), null=True, blank=True, max_length=256)
+
     coordinates = LocationById()
     origin = LocationById()
     destination = LocationById()
@@ -116,7 +118,7 @@ class Report(models.Model):
         reviewers = tuple(self.get_reviewers_qs().values_list('pk', flat=True))
         send_report_notification.delay(pk=self.pk,
                                        title=self.title,
-                                       author=self.author.username,
+                                       author=self.author.username if self.author else "(none)",
                                        description=self.description,
                                        reviewers=reviewers)