import bisect import json import operator import random from functools import reduce from itertools import chain from typing import Type, Any, Union, Self, TypeVar, Generic, NamedTuple, TypeAlias from django.apps import apps from django.core import serializers from django.core.exceptions import FieldDoesNotExist from django.db.models import Model, Q, CharField, SlugField, DecimalField from django.db.models.fields import IntegerField, SmallIntegerField, PositiveIntegerField, PositiveSmallIntegerField from django.db.models.fields.reverse_related import ManyToOneRel, OneToOneRel from pydantic.config import ConfigDict from c3nav.api.schema import BaseSchema from c3nav.editor.operations import DatabaseOperationCollection, CreateObjectOperation, UpdateObjectOperation, \ DeleteObjectOperation, ClearManyToManyOperation, FieldValuesDict, ObjectReference, PreviousObjectCollection, \ DatabaseOperation, ObjectID, FieldName, ModelName, CreateMultipleObjectsOperation, UpdateManyToManyOperation from c3nav.mapdata.fields import I18nField class ChangedManyToMany(BaseSchema): cleared: bool = False added: list[ObjectID] = [] removed: list[ObjectID] = [] @property def __bool__(self): return not (self.cleared or self.added or self.removed) class ChangedObject(BaseSchema): obj: ObjectReference titles: dict[str, str] | None created: bool = False deleted: bool = False fields: FieldValuesDict = {} m2m_changes: dict[FieldName, ChangedManyToMany] = {} def __bool__(self): return self.created or self.deleted or self.fields or any(self.m2m_changes.values()) class OperationDependencyObjectExists(BaseSchema): model_config = ConfigDict(frozen=True) obj: ObjectReference class OperationDependencyUniqueValue(BaseSchema): model_config = ConfigDict(frozen=True) model: str field: FieldName value: Any class OperationDependencyNoProtectedReference(BaseSchema): model_config = ConfigDict(frozen=True) obj: ObjectReference OperationDependency = Union[ OperationDependencyObjectExists, OperationDependencyNoProtectedReference, OperationDependencyUniqueValue, ] # todo: switch to new syntax once pydantic supports it OperationT = TypeVar('OperationT', bound=DatabaseOperation) class SingleOperationWithDependencies(BaseSchema, Generic[OperationT]): uid: tuple operation: OperationT dependencies: set[OperationDependency] = set() @property def main_op(self) -> Self: return self class MergableOperationsWithDependencies(BaseSchema): main_op: Union[ SingleOperationWithDependencies[CreateObjectOperation], SingleOperationWithDependencies[UpdateObjectOperation], ] sub_ops: list[SingleOperationWithDependencies[UpdateObjectOperation]] @property def dependencies(self) -> set[OperationDependency]: return self.main_op.dependencies | reduce(operator.or_, (op.dependencies for op in self.sub_ops), set()) OperationWithDependencies = Union[ SingleOperationWithDependencies, MergableOperationsWithDependencies, ] class FoundObjectReference(BaseSchema): model_config = ConfigDict(frozen=True) obj: ObjectReference field: FieldName on_delete: str class DummyValue: pass class OperationSituation(BaseSchema): # operations done so far operations: list[DatabaseOperation] = [] # uids of operationswithdependencies that are included now operation_uids: frozenset[tuple] = frozenset() # remaining operations still to do remaining_operations_with_dependencies: list[OperationWithDependencies] = [] # objects that still need to be created before some remaining operation (or that were simply deleted in this run) missing_objects: dict[ModelName, set[ObjectID]] = {} # unique values relevant for these operations that are currently not free occupied_unique_values: dict[ModelName, dict[FieldName, dict[Any, ObjectID]]] = {} # references to objects that need to be removed for in this run obj_references: dict[ModelName, dict[ObjectID, set[FoundObjectReference]]] = {} def fulfils_dependency(self, dependency: OperationDependency) -> bool: if isinstance(dependency, OperationDependencyObjectExists): return dependency.obj.id not in self.missing_objects.get(dependency.obj.model, set()) if isinstance(dependency, OperationDependencyNoProtectedReference): return not any( (reference.on_delete == "PROTECT") for reference in self.obj_references.get(dependency.obj.model, {}).get(dependency.obj.id, ()) ) if isinstance(dependency, OperationDependencyUniqueValue): return dependency.value not in self.occupied_unique_values.get(dependency.model, {}).get( dependency.field, set() ) raise ValueError def fulfils_dependencies(self, dependencies: set[OperationDependency]) -> bool: return all(self.fulfils_dependency(dependency) for dependency in dependencies) class ChangedObjectProblems(BaseSchema): obj_does_not_exist: bool = False cant_create: bool = False protected_references: set[FoundObjectReference] = set() field_does_not_exist: set[FieldName] = set() m2m_val_does_not_exist: dict[FieldName, set[ObjectID]] = {} dummy_values: dict[FieldName, Any] = {} ref_doesnt_exist: set[FieldName] = set() unique_constraint: set[FieldName] = set() def clean(self) -> bool: """ Clean up data and return true if there's any problemls left """ self.m2m_val_does_not_exist = { field_name: ids for field_name, ids in self.m2m_val_does_not_exist.items() if ids } return ( self.obj_does_not_exist or self.cant_create or self.protected_references or self.field_does_not_exist or self.m2m_val_does_not_exist or self.dummy_values or self.ref_doesnt_exist or self.unique_constraint ) class ChangeProblems(BaseSchema): model_does_not_exist: set[ModelName] = set() objects: dict[ModelName, dict[ObjectID, ChangedObjectProblems]] = {} def clean(self): self.objects = { model_name: problem_objects for model_name, problem_objects in ( (model_name, {pk: obj for pk, obj in problem_objects.items() if obj.clean()}) for model_name, problem_objects in self.objects.items() ) if problem_objects } def get_object(self, obj: ObjectReference): return self.objects.setdefault(obj.model, {}).setdefault(obj.id, ChangedObjectProblems()) @property def any(self) -> bool: """ Are there any problems? """ return bool(self.model_does_not_exist or self.objects) # todo: what if model does not exist… class ChangedObjectCollection(BaseSchema): """ A collection of ChangedObject instances, sorted by model and id. Also stores a PreviousObjectCollection for comparison with the current state. Iterable as a list of ChangedObject instances. """ prev: PreviousObjectCollection = PreviousObjectCollection() objects: dict[ModelName, dict[ObjectID, ChangedObject]] = {} def __iter__(self): yield from chain(*(objects.values() for model, objects in self.objects.items())) def __len__(self): return sum(len(v) for v in self.objects.values()) def add_operations(self, operations: DatabaseOperationCollection): """ Add the given operations, creating/updating changed objects to represent the resulting state. """ # todo: if something is being changed back, remove it from thingy? self.prev.add_other(operations.prev) for operation in operations: changed_object = self.objects.setdefault(operation.obj.model, {}).get(operation.obj.id, None) if changed_object is None: # todo: titles should be better, probably titles = ( operation.fields.get("title", {}) if isinstance(operation, CreateObjectOperation) else self.prev.get(operation.obj).titles ) changed_object = ChangedObject(obj=operation.obj, titles=titles) self.objects[operation.obj.model][operation.obj.id] = changed_object if isinstance(operation, CreateObjectOperation): changed_object.created = True changed_object.fields.update(operation.fields) elif isinstance(operation, UpdateObjectOperation): model = apps.get_model('mapdata', operation.obj.model) for field_name, value in operation.fields.items(): field = model._meta.get_field(field_name) if isinstance(field, I18nField) and field_name in changed_object.fields: changed_object.fields[field_name] = { lang: val for lang, val in {**changed_object.fields[field_name], **value}.items() } else: changed_object.fields[field_name] = value elif isinstance(operation, DeleteObjectOperation): changed_object.deleted = True else: changed_m2m = changed_object.m2m_changes.get(operation.field, None) if changed_m2m is None: changed_m2m = ChangedManyToMany() changed_object.m2m_changes[operation.field] = changed_m2m if isinstance(operation, ClearManyToManyOperation): changed_m2m.cleared = True changed_m2m.added = [] changed_m2m.removed = [] else: changed_m2m.added = sorted((set(changed_m2m.added) | operation.add_values) - operation.remove_values) changed_m2m.removed = sorted((set(changed_m2m.removed) - operation.add_values) | operation.remove_values) def clean_and_complete_prev(self): # todo: what the heck was this function for? ids: dict[ModelName, set[ObjectID]] = {} for model_name, changed_objects in self.objects.items(): ids.setdefault(model_name, set()).update(set(changed_objects.keys())) model = apps.get_model("mapdata", model_name) relations: dict[FieldName, Type[Model]] = {field.name: field.related_model for field in model.get_fields() if field.is_relation} for obj in changed_objects.values(): for field_name, value in obj.fields.items(): related_model = relations.get(field_name, None) if related_model is None or value is None: continue ids.setdefault(related_model._meta.model_name, set()).add(value) for field_name, field_changes in obj.m2m_changes.items(): related_model = relations[field_name] if field_changes.added or field_changes.removed: ids.setdefault(related_model._meta.model_name, set()).update(field_changes.added) ids.setdefault(related_model._meta.model_name, set()).update(field_changes.removed) # todo: move this to some kind of "usage explanation" function, implement rest of this class OperationsWithDependencies(NamedTuple): obj_operations: list[OperationWithDependencies] m2m_operations: list[SingleOperationWithDependencies] @property def as_operations_with_dependencies(self) -> tuple[OperationsWithDependencies, ChangeProblems]: operations_with_dependencies = self.OperationsWithDependencies(obj_operations=[], m2m_operations=[]) problems = ChangeProblems() for model_name, changed_objects in self.objects.items(): try: model = apps.get_model("mapdata", model_name) except LookupError: problems.model_does_not_exist.add(model_name) continue for changed_obj in changed_objects.values(): base_dependencies: set[OperationDependency] = ( set() if changed_obj.created else {OperationDependencyObjectExists(obj=changed_obj.obj)} ) if changed_obj.deleted: if changed_obj.created: continue operations_with_dependencies.obj_operations.append( SingleOperationWithDependencies( uid=(changed_obj.obj, "delete"), operation=DeleteObjectOperation(obj=changed_obj.obj), dependencies=( base_dependencies | {OperationDependencyNoProtectedReference(obj=changed_obj.obj)} ), ), ) continue initial_fields = dict() obj_sub_operations: list[OperationWithDependencies] = [] for field_name, value in changed_obj.fields.items(): try: field = model._meta.get_field(field_name) except FieldDoesNotExist: problems.get_object(changed_obj.obj).field_does_not_exist.add(field_name) continue if value is None: initial_fields[field_name] = None continue dependencies = base_dependencies.copy() # todo: prev if field.is_relation and not field.many_to_many: dependencies.add(OperationDependencyObjectExists(obj=ObjectReference( model=field.related_model._meta.model_name, id=value, ))) if field.unique: dependencies.add(OperationDependencyUniqueValue( model=model._meta.model_name, field=field_name, value=value, )) if not dependencies: initial_fields[field_name] = value continue initial_fields[field_name] = None if field.null else DummyValue obj_sub_operations.append(SingleOperationWithDependencies( uid=(changed_obj.obj, f"field_{field_name}"), operation=UpdateObjectOperation(obj=changed_obj.obj, fields={field_name: value}), dependencies=dependencies )) obj_main_operation = SingleOperationWithDependencies( uid=(changed_obj.obj, f"main"), operation=(CreateObjectOperation if changed_obj.created else UpdateObjectOperation)( obj=changed_obj.obj, fields=initial_fields, ), dependencies=base_dependencies, ) if not obj_sub_operations: operations_with_dependencies.obj_operations.append(obj_main_operation) else: operations_with_dependencies.obj_operations.append(MergableOperationsWithDependencies( main_op=obj_main_operation, sub_ops=obj_sub_operations, )) for field_name, m2m_changes in changed_obj.m2m_changes.items(): if m2m_changes.cleared: operations_with_dependencies.m2m_operations.append(SingleOperationWithDependencies( uid=(changed_obj.obj, f"m2mclear_{field_name}"), operation=ClearManyToManyOperation( obj=changed_obj.obj, field=field_name, ), dependencies={OperationDependencyObjectExists(obj=changed_obj.obj)}, )) if m2m_changes.added or m2m_changes.removed: operations_with_dependencies.m2m_operations.append(SingleOperationWithDependencies( uid=(changed_obj.obj, f"m2mupdate_{field_name}"), operation=UpdateManyToManyOperation( obj=changed_obj.obj, field=field_name, add_values=m2m_changes.added, remove_values=m2m_changes.removed, ), dependencies={OperationDependencyObjectExists(obj=changed_obj.obj)}, )) return operations_with_dependencies, problems class CreateStartOperationResult(NamedTuple): situation: OperationSituation unique_values_needed: dict[ModelName, dict[FieldName: set]] m2m_operations: list[SingleOperationWithDependencies] problems: ChangeProblems def create_start_operation_situation(self) -> CreateStartOperationResult: operations_with_dependencies, problems = self.as_operations_with_dependencies start_situation = OperationSituation( remaining_operations_with_dependencies=operations_with_dependencies.obj_operations ) referenced_objects: dict[ModelName, set[ObjectID]] = {} # objects that need to exist before deleted_existing_objects: dict[ModelName, set[ObjectID]] = {} # objects that need to exist before unique_values_needed: dict[ModelName, dict[FieldName: set]] = {} for operation in operations_with_dependencies.obj_operations: for dependency in operation.dependencies: if isinstance(dependency, OperationDependencyObjectExists): referenced_objects.setdefault(dependency.obj.model, set()).add(dependency.obj.id) elif isinstance(dependency, OperationDependencyUniqueValue): unique_values_needed.setdefault(dependency.model, {}).setdefault( dependency.field, set() ).add(dependency.value) elif isinstance(dependency, OperationDependencyNoProtectedReference): deleted_existing_objects.setdefault(dependency.obj.model, set()).add(dependency.obj.id) # references from m2m changes need also to be checked if they exist for model_name, changed_objects in self.objects.items(): try: model = apps.get_model("mapdata", model_name) except LookupError: # would already have been reported above continue # todo: how do we want m2m to work when it's cleared by the user but things were added in the meantime? for changed_obj in changed_objects.values(): for field_name, m2m_changes in changed_obj.m2m_changes.items(): try: field = model._meta.get_field(field_name) except FieldDoesNotExist: problems.get_object(changed_obj.obj).field_does_not_exist.add(field_name) continue referenced_objects.setdefault( field.related_model._meta.model_name, set() ).update(set(m2m_changes.added + m2m_changes.removed)) # let's find which objects that need to exist before actually exist for model, ids in referenced_objects.items(): model_cls = apps.get_model('mapdata', model) ids_found = set(model_cls.objects.filter(pk__in=ids).values_list('pk', flat=True)) start_situation.missing_objects[model] = {id_ for id_ in ids if id_ not in ids_found} # let's find which unique values are actually occupied right now for model, fields in unique_values_needed.items(): model_cls = apps.get_model('mapdata', model) q = Q() for field_name, values in fields.items(): q |= Q(**{f'{field_name}__in': values}) start_situation.occupied_unique_values[model] = {} for result in model_cls.objects.filter(q).values("id", *fields.keys()): pk = result.pop("id") for field_name, value in result.items(): if value in fields[field_name]: start_situation.occupied_unique_values[model].setdefault(field_name, {})[value] = pk # let's find which protected references to objects we want to delete have potential_fields: dict[ModelName, dict[FieldName, dict[ModelName, set[ObjectID]]]] = {} for model, ids in deleted_existing_objects.items(): # don't check this for objects that don't exist anymore ids -= start_situation.missing_objects.get(model, set()) for field in apps.get_model('mapdata', model)._meta.get_fields(): if (not isinstance(field, (ManyToOneRel, OneToOneRel)) or field.related_model._meta.app_label != "mapdata"): continue potential_fields.setdefault(field.related_model._meta.model_name, {}).setdefault(field.field.attname, {})[model] = ids # collect all references to objects we want to delete for model, fields in potential_fields.items(): model_cls = apps.get_model('mapdata', model) q = Q() targets_reverse: dict[FieldName, dict[ObjectID, ModelName]] = {} for field_name, targets in fields.items(): ids = reduce(operator.or_, targets.values(), set()) q |= Q(**{f'{field_name}__in': ids}) targets_reverse[field_name] = dict(chain(*(((id_, target_model) for id_ in target_ids) for target_model, target_ids in targets.items()))) for result in model_cls.objects.filter(q).values("id", *fields.keys()): source_ref = ObjectReference(model=model, id=result.pop("id")) for field, target_id in result.items(): target_model = targets_reverse[field][target_id] start_situation.obj_references.setdefault(target_model, {}).setdefault(target_id, set()).add( FoundObjectReference(obj=source_ref, field=field, on_delete=model_cls._meta.get_field(field).remote_field.on_delete.__name__) ) return self.CreateStartOperationResult( situation=start_situation, unique_values_needed=unique_values_needed, m2m_operations=operations_with_dependencies.m2m_operations, problems=problems, ) class ChangesAsOperations(NamedTuple): operations: DatabaseOperationCollection problems: ChangeProblems @property def as_operations(self) -> ChangesAsOperations: current_objects = {} for model_name, changed_objects in self.objects.items(): model = apps.get_model("mapdata", model_name) current_objects[model_name] = { obj["pk"]: obj["fields"] for obj in json.loads( serializers.serialize("json", model.objects.filter(pk__in=changed_objects.keys())) ) } start_situation, unique_values_needed, m2m_operations, problems = self.create_start_operation_situation() # situations still to deal with, sorted by number of operations open_situations: list[OperationSituation] = [start_situation] # situation that solves for all operations done_situation: OperationSituation | None = None # situations that ended prematurely ended_situations: list[OperationSituation] = [] # situations already encountered by set of operation uuids included, values are number of operations best_uids: dict[frozenset[tuple], int] = {} # unique values in db [only want to check for them once] dummy_unique_value_avoid: dict[ModelName, dict[FieldName, frozenset]] = {} available_model_ids: dict[ModelName, frozenset] = {} if not start_situation.remaining_operations_with_dependencies: # nothing to do? then we're done done_situation = start_situation while open_situations and not done_situation: situation = open_situations.pop(0) continued = False for i, remaining_operation in enumerate(situation.remaining_operations_with_dependencies): # check if the main operation can be run if not situation.fulfils_dependencies(remaining_operation.main_op.dependencies): continue # determine changes to state new_operation = remaining_operation.main_op.operation new_remaining_operations = [] uids_to_add: set[tuple] = set(remaining_operation.main_op.uid) if isinstance(remaining_operation, MergableOperationsWithDependencies): # sub_ops to be merged into this one or become pending operations new_operation: Union[CreateObjectOperation, UpdateObjectOperation] for sub_op in remaining_operation.sub_ops: if situation.fulfils_dependencies(sub_op.dependencies): new_operation.fields.update(sub_op.operation.fields) uids_to_add.add(sub_op.uid) else: new_remaining_operations.append(sub_op) model_cls = apps.get_model('mapdata', new_operation.obj.model) if isinstance(new_operation, (CreateObjectOperation, UpdateObjectOperation)): couldnt_fill_dummy = False for field_name, value in tuple(new_operation.fields.items()): if value is DummyValue: # if there's a dummy value to fill, we need to find a dummy value field = model_cls._meta.get_field(field_name) if field.is_relation: # for a relation, we will try to find a valid other object to reference if available_model_ids.get(field.related_model._meta.model_name) is None: # find available model ids, we only query these once for less queries available_model_ids[field.related_model._meta.model_name] = frozenset( field.related_model.objects.values_list('pk', flat=True) ) if field.unique: # if the field is unique we need to fin a value that isn't occupied # and, to be sure, that we haven't used as a dummyvalue before if dummy_unique_value_avoid.get(new_operation.obj.model, {}).get(field_name) is None: dummy_unique_value_avoid.setdefault( new_operation.obj.model, {} )[field_name] = frozenset( model_cls.objects.values_list(field_name.attname, flat=True) ) | unique_values_needed.get(new_operation.obj.model, {}).get(field_name, set()) choices = ( available_model_ids[field.related_model._meta.model_name] - dummy_unique_value_avoid[new_operation.obj.model][field_name] - set( situation.occupied_unique_values[new_operation.obj.model][field_name].keys() ) ) else: choices = available_model_ids[field.related_model._meta.model_name] if not choices: couldnt_fill_dummy = True break dummy_value = next(iter(choices)) else: if field.unique: # otherwise, an non-relational field needs a unique value if dummy_unique_value_avoid.get(new_operation.obj.model, {}).get(field_name) is None: dummy_unique_value_avoid.setdefault( new_operation.obj.model, {} )[field_name] = frozenset( model_cls.objects.values_list(field_name, flat=True) ) | unique_values_needed.get(new_operation.obj.model, {}).get(field_name, set()) occupied = ( dummy_unique_value_avoid[new_operation.obj.model][field_name] - set( situation.occupied_unique_values[new_operation.obj.model][field_name].keys() ) ) else: # this shouldn't happen, because dummy values are only used by non-relation fields # for unique constraints raise NotImplementedError # generate a value that works if isinstance(field, (SlugField, CharField)): new_val = "dummyvalue" while new_val in occupied: new_val = "dummyvalue"+str(random.randrange(1, 10000000)) elif isinstance(field, (DecimalField, IntegerField, SmallIntegerField, PositiveIntegerField, PositiveSmallIntegerField)): new_val = 0 while new_val in occupied: new_val += 1 else: raise NotImplementedError dummy_value = new_val # store the dummyvalue so we can tell the user about it problems.get_object(new_operation.obj).dummy_values[field_name] = dummy_value else: # we have set this field to a non-dummy value, if we used one before we can forget it problems.get_object(new_operation.obj).dummy_values.pop(field_name, None) if couldnt_fill_dummy: continue # if we couldn't fill a dummy value this operation is not doable, skip it # construct new situation new_situation = situation.model_copy(deep=True) if isinstance(new_operation, CreateObjectOperation) and new_situation.operations: last_operation = new_situation.operations[-1] if (isinstance(last_operation, CreateObjectOperation) and last_operation.obj.model == new_operation.obj.model): new_situation.operations[-1] = CreateMultipleObjectsOperation( objects=[last_operation, new_operation], ) elif (isinstance(last_operation, CreateMultipleObjectsOperation) and last_operation.objects[-1].obj.model == new_operation.obj.model): last_operation.objects.append(new_operation) else: new_situation.operations.append(new_operation) else: if not (isinstance(new_operation, UpdateObjectOperation) and not new_operation.fields): # we might have empty update operations, those can be ignored new_situation.operations.append(new_operation) new_situation.remaining_operations_with_dependencies.pop(i) new_situation.remaining_operations_with_dependencies.extend(new_remaining_operations) new_situation.operation_uids = new_situation.operation_uids | uids_to_add # even if we don't actually continue cause better paths existed, this situation is not a deadlock continued = True if not new_situation.remaining_operations_with_dependencies: # nothing left to do, congratulations we did it! done_situation = new_situation break if best_uids.get(new_situation.operation_uids, 1000000) <= len(new_situation.operations): # we already reached this situation with the same or less amount of operations continue if isinstance(new_operation, CreateObjectOperation): # if an object was created it's no longer missing new_situation.missing_objects.get(new_operation.obj.model, set()).discard(new_operation.obj.id) if isinstance(new_operation, UpdateObjectOperation): occupied_unique_values = new_situation.occupied_unique_values.get(new_operation.obj.model, {}) relations_changed = set() for field_name in new_operation.fields: field = model_cls._meta.get_field(field_name) if field.unique: # unique field was changed? remove unique value entry [might be readded below] occupied_unique_values[field_name] = { val: pk for val, pk in occupied_unique_values.get(field_name, {}).items() if pk != new_operation.obj.model } if field.is_relation: relations_changed.add(field_name) if relations_changed: # relation field was changed? remove reference entry [might be readded below] for model_name, references in tuple(new_situation.obj_references.items()): new_situation.obj_references[model_name] = { pk: ref for pk, ref in references.items() if ref.obj != new_operation.obj or ref.field not in relations_changed } if isinstance(new_operation, DeleteObjectOperation): # if an object was deleted it will now be missing new_situation.missing_objects.get(new_operation.obj.model, set()).add(new_operation.obj.id) # all unique values it occupied will no longer be occupied occupied_unique_values = new_situation.occupied_unique_values.get(new_operation.obj.model, {}) for field_name, values in tuple(occupied_unique_values.items()): occupied_unique_values[field_name] = {val: pk for val, pk in values.items() if pk != new_operation.obj.model} # all references that came from it, will no longer exist for model_name, references in tuple(new_situation.obj_references.items()): new_situation.obj_references[model_name] = { pk: {ref for ref in refs if ref.obj != new_operation.obj} for pk, refs in references.items() } # todo: we ignore cascading for now, do we want to keep it that way? else: for field_name, value in new_operation.fields.items(): field = model_cls._meta.get_field(field_name) if value is None: continue if field.unique: # unique field was changed? add unique value entry new_situation.occupied_unique_values.setdefault( new_operation.obj.model, {} ).setdefault(field_name, {})[value] = new_operation.obj.id if field.is_relation and not field.many_to_many: # relation field was changed? add foundobjectreference new_situation.obj_references.setdefault( field.related_model._meta.model_name, {} ).setdefault(value, set()).add( FoundObjectReference( obj=new_operation.obj, field=field_name, on_delete=field.remote_field.on_delete.__name__, ) ) # finally insert new situation bisect.insort(open_situations, new_situation, key=lambda s: len(s.operations)) best_uids[new_situation.operation_uids] = len(new_situation.operations) if not continued: ended_situations.append(situation) if not done_situation: done_situation = max(ended_situations, key=lambda s: (len(s.operation_uuids), -len(s.operations))) # add m2m for m2m_operation_with_dependencies in m2m_operations: if not done_situation.fulfils_dependencies(m2m_operation_with_dependencies.dependencies): done_situation.remaining_operations_with_dependencies.append(m2m_operation_with_dependencies) continue done_situation.operations.append(m2m_operation_with_dependencies.operation) for remaining_operation in done_situation.remaining_operations_with_dependencies: model_cls = apps.get_model("mapdata", remaining_operation.main_op.obj.model) obj = remaining_operation.main_op.obj problem_obj = problems.get_object(obj) if obj.id in done_situation.missing_objects.get(obj.model, set()): problem_obj.obj_does_not_exist = True continue if isinstance(remaining_operation.main_op, DeleteObjectOperation): problem_obj.protected_references = { found_ref for found_ref in done_situation.obj_references.get( remaining_operation.main_op.obj.model, {} )[remaining_operation.main_op.obj.id] if found_ref.on_delete == "PROTECT" } # this will fail if there are no protected references because that should never happen continue if isinstance(remaining_operation.main_op, CreateObjectOperation): problem_obj.cant_create = True sub_ops = (chain((remaining_operation.main_op,), remaining_operation.sub_ops) if isinstance(remaining_operation, MergableOperationsWithDependencies) else (remaining_operation.main_op,)) for sub_op in sub_ops: if isinstance(sub_op, UpdateManyToManyOperation): related_model_name = model_cls._meta.get_field(sub_op.field).related_model._meta.model_name missing_ids = ( done_situation.missing_objects.get(related_model_name, set()) & (sub_op.add_values | sub_op.remove_values) ) if missing_ids: problem_obj[sub_op.field] = missing_ids elif isinstance(sub_op, UpdateObjectOperation): for dependency in sub_op.dependencies: if isinstance(dependency, OperationDependencyObjectExists) and dependency.obj != sub_op.obj: problem_obj.ref_doesnt_exist.update(set(sub_op.fields.keys())) elif isinstance(dependency, OperationDependencyUniqueValue): problem_obj.unique_constraint.update(set(sub_op.fields.keys())) problems.clean() return self.ChangesAsOperations( operations=DatabaseOperationCollection( prev=self.prev, operations=done_situation.operations, ), problems=problems )