importhub now with automatic reports doing

This commit is contained in:
Laura Klünder 2023-12-25 23:38:23 +01:00
parent 04f7c27ecb
commit 4919b7d2cb
3 changed files with 155 additions and 3 deletions

View file

@ -1,3 +1,5 @@
import hashlib
import json
from uuid import UUID
import requests
@ -11,6 +13,7 @@ from shapely.geometry import shape
from c3nav.api.utils import NonEmptyStr
from c3nav.mapdata.models import Area, Space, LocationGroup, LocationSlug, MapUpdate
from c3nav.mapdata.models.geometry.space import POI
from c3nav.mapdata.models.report import Report
from c3nav.mapdata.utils.cache.changes import changed_geometries
@ -45,6 +48,15 @@ class Command(BaseCommand):
self.do_import(r.json())
MapUpdate.objects.create(type='importhub')
def do_report(self, prefix: str, obj_id: str, obj, report: Report):
import_prefix = f"{prefix}:{obj_id}:"
import_tag = import_prefix+hashlib.md5(str(obj).encode()).hexdigest()
Report.objects.filter(import_tag__startswith=import_prefix, open=True).exclude(import_tag=import_tag).delete()
if not Report.objects.filter(import_tag=import_tag).exists():
report.import_tag = import_tag
report.save()
report.notify_reviewers()
def do_import(self, data):
items: list[HubImportItem] = [HubImportItem.model_validate(item) for item in data]
items_by_id = {item.id: item for item in items}
@ -118,8 +130,31 @@ class Command(BaseCommand):
if not isinstance(result, target_type):
# need to change from POI to Area or inverted
if result.import_block_geom:
print(f"ERROR: {item.slug} / {item.id} needs to be switched to {target_type} but is blocked")
self.do_report(
prefix='hub:switch_type',
obj_id=str(item.id),
report=Report(
category="location-issue",
title=f"importhub: change geometry type for {result.title}, is blocked",
description=f"the object has a wrong geometry type and needs to be switched to "
f"{target_type} but the geometry is blocked",
location=result,
)
)
print(f"REPORT: {item.slug} / {item.id} needs to be switched to {target_type} but is blocked")
continue
self.do_report(
prefix='hub:switch_type',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: change geometry type for {result.title}, not implemented",
description=f"the object has a wrong geometry type and needs to be switched to "
f"{target_type} but this is not implemented yet",
location=result,
)
)
print(f"ERROR: {item.slug} / {item.id} needs to be switched to {target_type} but not implemented")
continue
@ -140,14 +175,18 @@ class Command(BaseCommand):
import_tag=import_tag,
)
geometry_needs_change = []
if result.space_id != new_space.pk:
if result.import_block_geom:
geometry_needs_change.append(f"change to space {new_space.title}")
print(f"NOTE: {item.slug} / {item.id} space has changed but is blocked")
else:
result.space_id = new_space.pk
if result.geometry != new_geometry or True:
if result.import_block_geom:
geometry_needs_change.append(f"change geometry")
print(f"NOTE: {item.slug} / {item.id} geometry has changed but is blocked")
else:
result.geometry = new_geometry
@ -156,16 +195,60 @@ class Command(BaseCommand):
new_main_point = Point(item.location) if item.location else None
if result.main_point != new_main_point:
if result.import_block_geom:
geometry_needs_change.append(f"change main point")
print(f"NOTE: {item.slug} / {item.id} main point has changed but is blocked")
else:
result.main_point = new_main_point
if geometry_needs_change:
self.do_report(
prefix='hub:change_geometry',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: geometry is blocked but needs changing",
description=f"changes needed: "+','.join(geometry_needs_change),
location=result,
)
)
data_needs_change = []
if item.slug != result.slug:
if result.import_block_data:
print(f"NOTE: {item.slug} / {item.id} slug has changed but is blocked")
data_needs_change.append(f"change slug to {item.slug}")
else:
if LocationSlug.objects.filter(slug=item.slug):
slug_occupied = LocationSlug.objects.filter(slug=item.slug).first()
if slug_occupied:
print(f"ERROR: {item.slug} / {item.id} slug {item.slug!r} is already occupied")
if is_new:
self.do_report(
prefix='hub:new_slug_occupied',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: want to import item with this slug ({item.slug}), occupied",
description=f"object to add {item.id}, for slug '{item.slug}' has name {item.name} "
f"and url {item.public_url} and should be in space {new_space.title}",
location=slug_occupied,
)
)
else:
self.do_report(
prefix='hub:new_slug_occupied',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: want change slug to {item.slug} but it's occupied",
description=f"object to add {item.id} for slug '{item.slug}' has name {item.name} "
f"and url {item.public_url}",
location=result,
)
)
continue
else:
result.slug = item.slug
@ -174,15 +257,30 @@ class Command(BaseCommand):
if new_titles != result.titles:
if result.import_block_data:
print(f"NOTE: {item.slug} / {item.id} name has changed but is blocked")
data_needs_change.append(f"change name to {item.name}")
else:
result.titles = new_titles
if item.public_url != result.external_url:
if result.import_block_data:
print(f"NOTE: {item.slug} / {item.id} external url has changed but is blocked")
data_needs_change.append(f"change external_url to {item.public_url}")
else:
result.external_url = item.public_url
if data_needs_change:
self.do_report(
prefix='hub:change_data',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: data is blocked but needs changing",
description=f"changes needed: "+','.join(data_needs_change),
location=result,
)
)
# time to check the groups
new_group_ids = set(group.id for group in new_groups)
if is_new:
@ -192,6 +290,17 @@ class Command(BaseCommand):
else:
if not new_group_ids:
print(f"SKIPPING: {item.slug} / {item.id} no longer has any group IDs, {hub_types}")
self.do_report(
prefix='hub:new_groups',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: location no longer has any valid group ids",
description=f"from the hub we would remove all groups, this seems wrong",
location=result,
)
)
continue
result.save()
@ -200,10 +309,33 @@ class Command(BaseCommand):
else:
old_group_ids = set(group.pk for group in result.groups.all())
if new_group_ids != old_group_ids:
self.do_report(
prefix='hub:new_groups',
obj_id=str(item.id),
obj=item,
report=Report(
category="location-issue",
title=f"importhub: new groups",
description=(f"hub wants new groups for this, groups are now: " +
str([group.title for group in new_groups])),
location=result,
)
)
print(f"NOTE: {item.slug} / {item.id} groups have changed, was " +
str([group.title for group in result.groups.all()]) +
", is now"+str([group.title for group in new_groups]), new_group_ids, old_group_ids)
for import_tag, location in locations_so_far.items():
self.do_report(
prefix='hub:new_groups',
obj_id=import_tag,
obj=import_tag,
report=Report(
category="location-issue",
title=f"importhub: delete this",
description=f"hub wants to delete this",
location=location,
)
)
print(f"NOTE: {location.slug} / {import_tag} should be deleted")