import bisect import json import operator import random from functools import reduce from itertools import chain from typing import Type, Any, Union, Self, TypeVar, Generic, NamedTuple from django.apps import apps from django.core import serializers from django.core.exceptions import FieldDoesNotExist from django.db.models import Model, Q, CharField, SlugField, DecimalField from django.db.models.fields import IntegerField, SmallIntegerField, PositiveIntegerField, PositiveSmallIntegerField from django.db.models.fields.reverse_related import ManyToOneRel, OneToOneRel from pydantic.config import ConfigDict from c3nav.api.schema import BaseSchema from c3nav.editor.operations import DatabaseOperationCollection, CreateObjectOperation, UpdateObjectOperation, \ DeleteObjectOperation, ClearManyToManyOperation, FieldValuesDict, ObjectReference, PreviousObjectCollection, \ DatabaseOperation, ObjectID, FieldName, ModelName, CreateMultipleObjectsOperation, UpdateManyToManyOperation from c3nav.mapdata.fields import I18nField from c3nav.mapdata.models import LocationSlug class ChangedManyToMany(BaseSchema): cleared: bool = False added: list[ObjectID] = [] removed: list[ObjectID] = [] @property def __bool__(self): return not (self.cleared or self.added or self.removed) class ChangedObject(BaseSchema): obj: ObjectReference titles: dict[str, str] | None created: bool = False deleted: bool = False fields: FieldValuesDict = {} m2m_changes: dict[FieldName, ChangedManyToMany] = {} def __bool__(self): return self.created or self.deleted or self.fields or any(self.m2m_changes.values()) class OperationDependencyObjectExists(BaseSchema): model_config = ConfigDict(frozen=True) obj: ObjectReference class OperationDependencyUniqueValue(BaseSchema): model_config = ConfigDict(frozen=True) model: str field: FieldName value: Any class OperationDependencyNoProtectedReference(BaseSchema): model_config = ConfigDict(frozen=True) obj: ObjectReference OperationDependency = Union[ OperationDependencyObjectExists, OperationDependencyNoProtectedReference, OperationDependencyUniqueValue, ] # todo: switch to new syntax once pydantic supports it OperationT = TypeVar('OperationT', bound=DatabaseOperation) class SingleOperationWithDependencies(BaseSchema, Generic[OperationT]): uid: tuple operation: OperationT dependencies: set[OperationDependency] = set() @property def main_op(self) -> Self: return self class MergableOperationsWithDependencies(BaseSchema): main_op: Union[ SingleOperationWithDependencies[CreateObjectOperation], SingleOperationWithDependencies[UpdateObjectOperation], ] sub_ops: list[SingleOperationWithDependencies[UpdateObjectOperation]] @property def dependencies(self) -> set[OperationDependency]: return self.main_op.dependencies | reduce(operator.or_, (op.dependencies for op in self.sub_ops), set()) OperationWithDependencies = Union[ SingleOperationWithDependencies, MergableOperationsWithDependencies, ] class FoundObjectReference(BaseSchema): model_config = ConfigDict(frozen=True) obj: ObjectReference field: FieldName on_delete: str class DummyValue: pass class OperationSituation(BaseSchema): # operations done so far operations: list[DatabaseOperation] = [] # uids of operationswithdependencies that are included now operation_uids: frozenset[tuple] = frozenset() # remaining operations still to do remaining_operations_with_dependencies: list[OperationWithDependencies] = [] # objects that still need to be created before some remaining operation (True = missing) missing_objects: dict[ModelName, dict[ObjectID, bool]] = {} # unique values relevant for these operations that are currently not free occupied_unique_values: dict[ModelName, dict[FieldName, dict[Any, ObjectID | None]]] = {} # references to objects that need to be removed for in this run obj_references: dict[ModelName, dict[ObjectID, set[FoundObjectReference]]] = {} @property def dependency_snapshot(self): return ( frozenset(chain(*( ((model_name, pk) for pk, missing in objects.items() if missing) for model_name, objects in self.missing_objects.items() ))), frozenset( chain(*( chain(*( ((model_name, field_name, field_value) for field_value, pk in values.items() if pk is not None) for field_name, values in fields.items() )) for model_name, fields in self.occupied_unique_values.items() )) ), frozenset( chain(*( chain(*( ((model_name, pk, found_ref) for found_ref in found_refs) for pk, found_refs in objects.items() )) for model_name, objects in self.obj_references.items() )) ) ) def fulfils_dependency(self, dependency: OperationDependency) -> bool: if isinstance(dependency, OperationDependencyObjectExists): return not self.missing_objects.get(dependency.obj.model, {}).get(dependency.obj.id, False) if isinstance(dependency, OperationDependencyNoProtectedReference): return not any( (reference.on_delete == "PROTECT") for reference in self.obj_references.get(dependency.obj.model, {}).get(dependency.obj.id, ()) ) if isinstance(dependency, OperationDependencyUniqueValue): return self.occupied_unique_values.get(dependency.model, {}).get( dependency.field, {} ).get(dependency.value, None) is None raise ValueError def fulfils_dependencies(self, dependencies: set[OperationDependency]) -> bool: return all(self.fulfils_dependency(dependency) for dependency in dependencies) class ChangedObjectProblems(BaseSchema): obj_does_not_exist: bool = False cant_create: bool = False protected_references: set[FoundObjectReference] = set() field_does_not_exist: set[FieldName] = set() m2m_val_does_not_exist: dict[FieldName, set[ObjectID]] = {} dummy_values: dict[FieldName, Any] = {} ref_doesnt_exist: set[FieldName] = set() unique_constraint: set[FieldName] = set() def clean(self) -> bool: """ Clean up data and return true if there's any problemls left """ self.m2m_val_does_not_exist = { field_name: ids for field_name, ids in self.m2m_val_does_not_exist.items() if ids } return ( self.obj_does_not_exist or self.cant_create or self.protected_references or self.field_does_not_exist or self.m2m_val_does_not_exist or self.dummy_values or self.ref_doesnt_exist or self.unique_constraint ) class ChangeProblems(BaseSchema): model_does_not_exist: set[ModelName] = set() objects: dict[ModelName, dict[ObjectID, ChangedObjectProblems]] = {} def clean(self): self.objects = { model_name: problem_objects for model_name, problem_objects in ( (model_name, {pk: obj for pk, obj in problem_objects.items() if obj.clean()}) for model_name, problem_objects in self.objects.items() ) if problem_objects } def get_object(self, obj: ObjectReference): return self.objects.setdefault(obj.model, {}).setdefault(obj.id, ChangedObjectProblems()) @property def any(self) -> bool: """ Are there any problems? """ return bool(self.model_does_not_exist or self.objects) # todo: what if model does not exist… class ChangedObjectCollection(BaseSchema): """ A collection of ChangedObject instances, sorted by model and id. Also stores a PreviousObjectCollection for comparison with the current state. Iterable as a list of ChangedObject instances. """ prev: PreviousObjectCollection = PreviousObjectCollection() objects: dict[ModelName, dict[ObjectID, ChangedObject]] = {} def __iter__(self): yield from chain(*(objects.values() for model, objects in self.objects.items())) def __len__(self): return sum(len(v) for v in self.objects.values()) def add_operations(self, operations: DatabaseOperationCollection): """ Add the given operations, creating/updating changed objects to represent the resulting state. """ # todo: if something is being changed back, remove it from thingy? self.prev.add_other(operations.prev) for operation in operations: changed_object = self.objects.setdefault(operation.obj.model, {}).get(operation.obj.id, None) if changed_object is None: # todo: titles should be better, probably titles = ( operation.fields.get("title", {}) if isinstance(operation, CreateObjectOperation) else self.prev.get(operation.obj).titles ) changed_object = ChangedObject(obj=operation.obj, titles=titles) self.objects[operation.obj.model][operation.obj.id] = changed_object if isinstance(operation, CreateObjectOperation): changed_object.created = True changed_object.fields.update(operation.fields) elif isinstance(operation, UpdateObjectOperation): model = apps.get_model('mapdata', operation.obj.model) for field_name, value in operation.fields.items(): field = model._meta.get_field(field_name) if isinstance(field, I18nField) and field_name in changed_object.fields: changed_object.fields[field_name] = { lang: val for lang, val in {**changed_object.fields[field_name], **value}.items() } else: changed_object.fields[field_name] = value elif isinstance(operation, DeleteObjectOperation): changed_object.deleted = True else: changed_m2m = changed_object.m2m_changes.get(operation.field, None) if changed_m2m is None: changed_m2m = ChangedManyToMany() changed_object.m2m_changes[operation.field] = changed_m2m if isinstance(operation, ClearManyToManyOperation): changed_m2m.cleared = True changed_m2m.added = [] changed_m2m.removed = [] else: changed_m2m.added = sorted((set(changed_m2m.added) | operation.add_values) - operation.remove_values) changed_m2m.removed = sorted((set(changed_m2m.removed) - operation.add_values) | operation.remove_values) def clean_and_complete_prev(self): # todo: what the heck was this function for? ids: dict[ModelName, set[ObjectID]] = {} for model_name, changed_objects in self.objects.items(): ids.setdefault(model_name, set()).update(set(changed_objects.keys())) model = apps.get_model("mapdata", model_name) relations: dict[FieldName, Type[Model]] = {field.name: field.related_model for field in model.get_fields() if field.is_relation} for obj in changed_objects.values(): for field_name, value in obj.fields.items(): related_model = relations.get(field_name, None) if related_model is None or value is None: continue ids.setdefault(related_model._meta.model_name, set()).add(value) for field_name, field_changes in obj.m2m_changes.items(): related_model = relations[field_name] if field_changes.added or field_changes.removed: ids.setdefault(related_model._meta.model_name, set()).update(field_changes.added) ids.setdefault(related_model._meta.model_name, set()).update(field_changes.removed) # todo: move this to some kind of "usage explanation" function, implement rest of this class OperationsWithDependencies(NamedTuple): obj_operations: list[OperationWithDependencies] m2m_operations: list[SingleOperationWithDependencies] @property def as_operations_with_dependencies(self) -> tuple[OperationsWithDependencies, ChangeProblems]: operations_with_dependencies = self.OperationsWithDependencies(obj_operations=[], m2m_operations=[]) problems = ChangeProblems() for model_name, changed_objects in self.objects.items(): try: model = apps.get_model("mapdata", model_name) except LookupError: problems.model_does_not_exist.add(model_name) continue for changed_obj in changed_objects.values(): base_dependencies: set[OperationDependency] = ( set() if changed_obj.created else {OperationDependencyObjectExists(obj=changed_obj.obj)} ) if changed_obj.deleted: if changed_obj.created: continue operations_with_dependencies.obj_operations.append( SingleOperationWithDependencies( uid=(changed_obj.obj, "delete"), operation=DeleteObjectOperation(obj=changed_obj.obj), dependencies=( base_dependencies | {OperationDependencyNoProtectedReference(obj=changed_obj.obj)} ), ), ) continue initial_fields = dict() obj_sub_operations: list[OperationWithDependencies] = [] for field_name, value in changed_obj.fields.items(): try: field = model._meta.get_field(field_name) except FieldDoesNotExist: problems.get_object(changed_obj.obj).field_does_not_exist.add(field_name) continue if value is None: initial_fields[field_name] = None continue dependencies = base_dependencies.copy() # todo: prev if field.is_relation and not field.many_to_many: dependencies.add(OperationDependencyObjectExists(obj=ObjectReference( model=field.related_model._meta.model_name, id=value, ))) if field.unique: dependencies.add(OperationDependencyUniqueValue( model="locationslug" if issubclass(model, LocationSlug) else model._meta.model_name, field=field_name, value=value, )) if not dependencies: initial_fields[field_name] = value continue initial_fields[field_name] = None if field.null else DummyValue obj_sub_operations.append(SingleOperationWithDependencies( uid=(changed_obj.obj, f"field_{field_name}"), operation=UpdateObjectOperation(obj=changed_obj.obj, fields={field_name: value}), dependencies=dependencies )) obj_main_operation = SingleOperationWithDependencies( uid=(changed_obj.obj, f"main"), operation=(CreateObjectOperation if changed_obj.created else UpdateObjectOperation)( obj=changed_obj.obj, fields=initial_fields, ), dependencies=base_dependencies, ) if not obj_sub_operations: operations_with_dependencies.obj_operations.append(obj_main_operation) else: operations_with_dependencies.obj_operations.append(MergableOperationsWithDependencies( main_op=obj_main_operation, sub_ops=obj_sub_operations, )) for field_name, m2m_changes in changed_obj.m2m_changes.items(): if m2m_changes.cleared: operations_with_dependencies.m2m_operations.append(SingleOperationWithDependencies( uid=(changed_obj.obj, f"m2mclear_{field_name}"), operation=ClearManyToManyOperation( obj=changed_obj.obj, field=field_name, ), dependencies={OperationDependencyObjectExists(obj=changed_obj.obj)}, )) if m2m_changes.added or m2m_changes.removed: operations_with_dependencies.m2m_operations.append(SingleOperationWithDependencies( uid=(changed_obj.obj, f"m2mupdate_{field_name}"), operation=UpdateManyToManyOperation( obj=changed_obj.obj, field=field_name, add_values=m2m_changes.added, remove_values=m2m_changes.removed, ), dependencies={OperationDependencyObjectExists(obj=changed_obj.obj)}, )) return operations_with_dependencies, problems class CreateStartOperationResult(NamedTuple): situation: OperationSituation unique_values_needed: dict[ModelName, dict[FieldName: set]] m2m_operations: list[SingleOperationWithDependencies] problems: ChangeProblems def create_start_operation_situation(self) -> CreateStartOperationResult: operations_with_dependencies, problems = self.as_operations_with_dependencies start_situation = OperationSituation( remaining_operations_with_dependencies=operations_with_dependencies.obj_operations ) referenced_objects: dict[ModelName, set[ObjectID]] = {} # objects that need to exist before deleted_existing_objects: dict[ModelName, set[ObjectID]] = {} # objects that need to exist before unique_values_needed: dict[ModelName, dict[FieldName: set]] = {} for operation in operations_with_dependencies.obj_operations: for dependency in operation.dependencies: if isinstance(dependency, OperationDependencyObjectExists): referenced_objects.setdefault(dependency.obj.model, set()).add(dependency.obj.id) elif isinstance(dependency, OperationDependencyUniqueValue): unique_values_needed.setdefault(dependency.model, {}).setdefault( dependency.field, set() ).add(dependency.value) elif isinstance(dependency, OperationDependencyNoProtectedReference): deleted_existing_objects.setdefault(dependency.obj.model, set()).add(dependency.obj.id) # references from m2m changes need also to be checked if they exist for model_name, changed_objects in self.objects.items(): try: model = apps.get_model("mapdata", model_name) except LookupError: # would already have been reported above continue # todo: how do we want m2m to work when it's cleared by the user but things were added in the meantime? for changed_obj in changed_objects.values(): for field_name, m2m_changes in changed_obj.m2m_changes.items(): try: field = model._meta.get_field(field_name) except FieldDoesNotExist: problems.get_object(changed_obj.obj).field_does_not_exist.add(field_name) continue if field.related_model._meta.app_label != "mapdata": continue referenced_objects.setdefault( field.related_model._meta.model_name, set() ).update(set(m2m_changes.added + m2m_changes.removed)) # let's find which objects that need to exist before actually exist for model, ids in referenced_objects.items(): model_cls = apps.get_model('mapdata', model) ids_found = set(model_cls.objects.filter(pk__in=ids).values_list('pk', flat=True)) start_situation.missing_objects[model] = {id_: (id_ not in ids_found) for id_ in ids} # let's find which unique values are actually occupied right now for model, fields in unique_values_needed.items(): model_cls = apps.get_model('mapdata', model) q = Q() start_situation.occupied_unique_values[model] = {} for field_name, values in fields.items(): q |= Q(**{f'{field_name}__in': values}) start_situation.occupied_unique_values[model][field_name] = {value: None for value in values} start_situation.occupied_unique_values[model] = {} for result in model_cls.objects.filter(q).values("id", *fields.keys()): pk = result.pop("id") for field_name, value in result.items(): if value in fields[field_name]: start_situation.occupied_unique_values[model].setdefault(field_name, {})[value] = pk # let's find which protected references to objects we want to delete have potential_fields: dict[ModelName, dict[FieldName, dict[ModelName, set[ObjectID]]]] = {} for model, ids in deleted_existing_objects.items(): # don't check this for objects that don't exist anymore ids -= set(start_situation.missing_objects.get(model, {}).keys()) for field in apps.get_model('mapdata', model)._meta.get_fields(): if (not isinstance(field, (ManyToOneRel, OneToOneRel)) or field.related_model._meta.app_label != "mapdata"): continue potential_fields.setdefault(field.related_model._meta.model_name, {}).setdefault(field.field.attname, {})[model] = ids # collect all references to objects we want to delete for model, fields in potential_fields.items(): model_cls = apps.get_model('mapdata', model) q = Q() targets_reverse: dict[FieldName, dict[ObjectID, ModelName]] = {} for field_name, targets in fields.items(): ids = reduce(operator.or_, targets.values(), set()) q |= Q(**{f'{field_name}__in': ids}) targets_reverse[field_name] = dict(chain(*(((id_, target_model) for id_ in target_ids) for target_model, target_ids in targets.items()))) for result in model_cls.objects.filter(q).values("id", *fields.keys()): source_ref = ObjectReference(model=model, id=result.pop("id")) for field, target_id in result.items(): target_model = targets_reverse[field][target_id] start_situation.obj_references.setdefault(target_model, {}).setdefault(target_id, set()).add( FoundObjectReference(obj=source_ref, field=field, on_delete=model_cls._meta.get_field(field).remote_field.on_delete.__name__) ) return self.CreateStartOperationResult( situation=start_situation, unique_values_needed=unique_values_needed, m2m_operations=operations_with_dependencies.m2m_operations, problems=problems, ) class ChangesAsOperations(NamedTuple): operations: DatabaseOperationCollection problems: ChangeProblems @property def as_operations(self) -> ChangesAsOperations: current_objects = {} for model_name, changed_objects in self.objects.items(): model = apps.get_model("mapdata", model_name) current_objects[model_name] = { obj["pk"]: obj["fields"] for obj in json.loads( serializers.serialize("json", model.objects.filter(pk__in=changed_objects.keys())) ) } start_situation, unique_values_needed, m2m_operations, problems = self.create_start_operation_situation() # situations still to deal with, sorted by number of operations open_situations: list[OperationSituation] = [start_situation] # situation that solves for all operations done_situation: OperationSituation | None = None # situations that ended prematurely ended_situations: list[OperationSituation] = [] # best way to get to a certain dependency snapshot, values are number of operations best_dependency_snapshots: dict[tuple, int] = {} # unique values in db [only want to check for them once] dummy_unique_value_avoid: dict[ModelName, dict[FieldName, frozenset]] = {} available_model_ids: dict[ModelName, frozenset] = {} if not start_situation.remaining_operations_with_dependencies: # nothing to do? then we're done done_situation = start_situation num = 0 while open_situations and not done_situation: num += 1 if num > 1000: raise ValueError("as_operations might be in an endless loop") situation = open_situations.pop(0) continued = False for i, remaining_operation in enumerate(situation.remaining_operations_with_dependencies): # check if the main operation can be run if not situation.fulfils_dependencies(remaining_operation.main_op.dependencies): continue # determine changes to state new_operation = remaining_operation.main_op.operation new_remaining_operations = [] uids_to_add: set[tuple] = {remaining_operation.main_op.uid} if isinstance(remaining_operation, MergableOperationsWithDependencies): # sub_ops to be merged into this one or become pending operations new_operation: Union[CreateObjectOperation, UpdateObjectOperation] for sub_op in remaining_operation.sub_ops: if situation.fulfils_dependencies(sub_op.dependencies): new_operation.fields.update(sub_op.operation.fields) uids_to_add.add(sub_op.uid) else: new_remaining_operations.append(sub_op) model_cls = apps.get_model('mapdata', new_operation.obj.model) operation_model_name = ("locationslug" if issubclass(model_cls, LocationSlug) else new_operation.obj.model) if isinstance(new_operation, (CreateObjectOperation, UpdateObjectOperation)): couldnt_fill_dummy = False for field_name, value in tuple(new_operation.fields.items()): if value is DummyValue: # if there's a dummy value to fill, we need to find a dummy value field = model_cls._meta.get_field(field_name) if field.is_relation: # for a relation, we will try to find a valid other object to reference if available_model_ids.get(field.related_model._meta.model_name) is None: # find available model ids, we only query these once for less queries available_model_ids[field.related_model._meta.model_name] = frozenset( field.related_model.objects.values_list('pk', flat=True) ) if field.unique: # if the field is unique we need to find a value that isn't occupied # and, to be sure, that we haven't used as a dummyvalue before if dummy_unique_value_avoid.get(operation_model_name, {}).get(field_name) is None: dummy_unique_value_avoid.setdefault( operation_model_name, {} )[field_name] = frozenset( model_cls.objects.values_list(field_name.attname, flat=True) ) | unique_values_needed.get(operation_model_name, {}).get(field_name, set()) choices = ( available_model_ids[field.related_model._meta.model_name] - dummy_unique_value_avoid[operation_model_name][field_name] - set(val for val, id_ in situation.occupied_unique_values[ operation_model_name ][field_name].items() if id_ is not None) ) else: choices = available_model_ids[field.related_model._meta.model_name] if not choices: couldnt_fill_dummy = True break dummy_value = next(iter(choices)) else: if field.unique: # otherwise, an non-relational field needs a unique value if dummy_unique_value_avoid.get(operation_model_name, {}).get(field_name) is None: dummy_unique_value_avoid.setdefault( operation_model_name, {} )[field_name] = frozenset( model_cls.objects.values_list(field_name, flat=True) ) | unique_values_needed.get(operation_model_name, {}).get(field_name, set()) occupied = ( dummy_unique_value_avoid[operation_model_name][field_name] - set(val for val, id_ in situation.occupied_unique_values[ operation_model_name ][field_name].items() if id_ is not None) ) else: # this shouldn't happen, because dummy values are only used by non-relation fields # for unique constraints raise NotImplementedError # generate a value that works if isinstance(field, (SlugField, CharField)): new_val = "dummyvalue" while new_val in occupied: new_val = "dummyvalue"+str(random.randrange(1, 10000000)) elif isinstance(field, (DecimalField, IntegerField, SmallIntegerField, PositiveIntegerField, PositiveSmallIntegerField)): new_val = 0 while new_val in occupied: new_val += 1 else: raise NotImplementedError dummy_value = new_val # store the dummyvalue so we can tell the user about it problems.get_object(new_operation.obj).dummy_values[field_name] = dummy_value new_operation.fields[field_name] = dummy_value else: # we have set this field to a non-dummy value, if we used one before we can forget it problems.get_object(new_operation.obj).dummy_values.pop(field_name, None) if couldnt_fill_dummy: continue # if we couldn't fill a dummy value this operation is not doable, skip it # construct new situation new_situation = situation.model_copy(deep=True) model = apps.get_model('mapdata', new_operation.obj.model) for parent in model._meta.get_parent_list(): if parent._meta.concrete_model is not model._meta.concrete_model: is_multi_inheritance = True break else: is_multi_inheritance = False if (isinstance(new_operation, CreateObjectOperation) and new_situation.operations and not is_multi_inheritance): last_operation = new_situation.operations[-1] if (isinstance(last_operation, CreateObjectOperation) and last_operation.obj.model == new_operation.obj.model): new_situation.operations[-1] = CreateMultipleObjectsOperation( objects=[last_operation, new_operation], ) elif (isinstance(last_operation, CreateMultipleObjectsOperation) and len(last_operation.objects) < 25 and last_operation.objects[-1].obj.model == new_operation.obj.model): last_operation.objects.append(new_operation) else: new_situation.operations.append(new_operation) else: if not (isinstance(new_operation, UpdateObjectOperation) and not new_operation.fields): # we might have empty update operations, those can be ignored new_situation.operations.append(new_operation) new_situation.remaining_operations_with_dependencies.pop(i) new_situation.remaining_operations_with_dependencies.extend(new_remaining_operations) new_situation.operation_uids = new_situation.operation_uids | uids_to_add # even if we don't actually continue cause better paths existed, this situation is not a deadlock continued = True if not new_situation.remaining_operations_with_dependencies: # nothing left to do, congratulations we did it! done_situation = new_situation break dependency_snapshot = (new_situation.dependency_snapshot, len(new_situation.operation_uids)) if best_dependency_snapshots.get(dependency_snapshot, 1000000) <= len(new_situation.operations): # we already reached this dependency snapshot with the same number of operations # in a better way continue if isinstance(new_operation, CreateObjectOperation): # if an object was created it's no longer missing for mn in {new_operation.obj.model, operation_model_name}: missing_objects = new_situation.missing_objects.get(mn, {}) if new_operation.obj.id in missing_objects: missing_objects[new_operation.obj.id] = False if isinstance(new_operation, UpdateObjectOperation): occupied_unique_values = new_situation.occupied_unique_values.get(new_operation.obj.model, {}) relations_changed = set() for field_name in new_operation.fields: field = model_cls._meta.get_field(field_name) if field.unique: # unique field was changed? remove unique value entry [might be readded below] occupied_unique_values[field_name] = { val: (None if new_operation.obj.id == pk else pk) for val, pk in occupied_unique_values.get(field_name, {}).items() } if field.is_relation: relations_changed.add(field_name) if relations_changed: # relation field was changed? remove reference entry [might be readded below] for model_name, references in tuple(new_situation.obj_references.items()): new_situation.obj_references[model_name] = { pk: ref for pk, ref in references.items() if ref.obj != new_operation.obj or ref.field not in relations_changed } if isinstance(new_operation, DeleteObjectOperation): # if an object was deleted it will now be missing for mn in {new_operation.obj.model, operation_model_name}: missing_objects = new_situation.missing_objects.get(mn, {}) if new_operation.obj.id in missing_objects: missing_objects[new_operation.obj.id] = True # all unique values it occupied will no longer be occupied occupied_unique_values = new_situation.occupied_unique_values.get(new_operation.obj.model, {}) for field_name, values in tuple(occupied_unique_values.items()): occupied_unique_values[field_name] = {val: (None if new_operation.obj.id == pk else pk) for val, pk in values.items()} # all references that came from it, will no longer exist for model_name, references in tuple(new_situation.obj_references.items()): new_situation.obj_references[model_name] = { pk: {ref for ref in refs if ref.obj != new_operation.obj} for pk, refs in references.items() } # todo: we ignore cascading for now, do we want to keep it that way? probably not! else: for field_name, value in new_operation.fields.items(): field = model_cls._meta.get_field(field_name) if value is None: continue if field.unique: # unique field was changed? add unique value entry field_occupied_values = new_situation.occupied_unique_values.get( new_operation.obj.model, {} ).get(field_name, {}) if value in field_occupied_values: field_occupied_values[value] = new_operation.obj.id if field.is_relation and not field.many_to_many: # relation field was changed? add foundobjectreference model_refs = new_situation.obj_references.get(field.related_model._meta.model_name, {}) if value in model_refs: model_refs[value].add( FoundObjectReference( obj=new_operation.obj, field=field_name, on_delete=field.remote_field.on_delete.__name__, ) ) # finally insert new situation bisect.insort(open_situations, new_situation, key=lambda s: (len(s.operations), )) best_dependency_snapshots[dependency_snapshot] = len(new_situation.operations) if not continued: ended_situations.append(situation) if not done_situation: done_situation = max(ended_situations, key=lambda s: (len(s.operation_uids), -len(s.operations))) # add m2m for m2m_operation_with_dependencies in m2m_operations: if not done_situation.fulfils_dependencies(m2m_operation_with_dependencies.dependencies): done_situation.remaining_operations_with_dependencies.append(m2m_operation_with_dependencies) continue done_situation.operations.append(m2m_operation_with_dependencies.operation) for remaining_operation in done_situation.remaining_operations_with_dependencies: model_cls = apps.get_model("mapdata", remaining_operation.main_op.operation.obj.model) obj = remaining_operation.main_op.operation.obj problem_obj = problems.get_object(obj) if done_situation.missing_objects.get(obj.model, {}).get(obj.id): problem_obj.obj_does_not_exist = True continue if isinstance(remaining_operation.main_op, DeleteObjectOperation): problem_obj.protected_references = { found_ref for found_ref in done_situation.obj_references.get( remaining_operation.main_op.obj.model, {} )[remaining_operation.main_op.obj.id] if found_ref.on_delete == "PROTECT" } # this will fail if there are no protected references because that should never happen continue if isinstance(remaining_operation.main_op, CreateObjectOperation): problem_obj.cant_create = True sub_ops = (chain((remaining_operation.main_op,), remaining_operation.sub_ops) if isinstance(remaining_operation, MergableOperationsWithDependencies) else (remaining_operation.main_op,)) for sub_op in sub_ops: if isinstance(sub_op, UpdateManyToManyOperation): related_model_name = model_cls._meta.get_field(sub_op.field).related_model._meta.model_name missing_ids = ( set(id_ for id_, missing in done_situation.missing_objects.get(related_model_name, {}).items() if missing) & (sub_op.add_values | sub_op.remove_values) ) if missing_ids: problem_obj[sub_op.field] = missing_ids elif isinstance(sub_op, UpdateObjectOperation): for dependency in sub_op.dependencies: if isinstance(dependency, OperationDependencyObjectExists) and dependency.obj != sub_op.obj: problem_obj.ref_doesnt_exist.update(set(sub_op.fields.keys())) elif isinstance(dependency, OperationDependencyUniqueValue): problem_obj.unique_constraint.update(set(sub_op.fields.keys())) problems.clean() return self.ChangesAsOperations( operations=DatabaseOperationCollection( prev=self.prev, operations=done_situation.operations, ), problems=problems )