team-3/src/c3nav/editor/changes.py

import bisect
import json
import operator
import random
from functools import reduce
from itertools import chain
from typing import Type, Any, Union, Self, TypeVar, Generic, NamedTuple, TypeAlias

from django.apps import apps
from django.core import serializers
from django.core.exceptions import FieldDoesNotExist
from django.db.models import Model, Q, CharField, SlugField, DecimalField
from django.db.models.fields import IntegerField, SmallIntegerField, PositiveIntegerField, PositiveSmallIntegerField
from django.db.models.fields.reverse_related import ManyToOneRel, OneToOneRel
from pydantic.config import ConfigDict

from c3nav.api.schema import BaseSchema
from c3nav.editor.operations import DatabaseOperationCollection, CreateObjectOperation, UpdateObjectOperation, \
    DeleteObjectOperation, ClearManyToManyOperation, FieldValuesDict, ObjectReference, PreviousObjectCollection, \
    DatabaseOperation, ObjectID, FieldName, ModelName, CreateMultipleObjectsOperation, UpdateManyToManyOperation
from c3nav.mapdata.fields import I18nField


class ChangedManyToMany(BaseSchema):
    cleared: bool = False
    added: list[ObjectID] = []
    removed: list[ObjectID] = []

    @property
    def __bool__(self):
        return not (self.cleared or self.added or self.removed)


class ChangedObject(BaseSchema):
    obj: ObjectReference
    titles: dict[str, str] | None
    created: bool = False
    deleted: bool = False
    fields: FieldValuesDict = {}
    m2m_changes: dict[FieldName, ChangedManyToMany] = {}

    def __bool__(self):
        return self.created or self.deleted or self.fields or any(self.m2m_changes.values())


class OperationDependencyObjectExists(BaseSchema):
    model_config = ConfigDict(frozen=True)

    obj: ObjectReference


class OperationDependencyUniqueValue(BaseSchema):
    model_config = ConfigDict(frozen=True)

    model: str
    field: FieldName
    value: Any


class OperationDependencyNoProtectedReference(BaseSchema):
    model_config = ConfigDict(frozen=True)

    obj: ObjectReference


OperationDependency = Union[
    OperationDependencyObjectExists,
    OperationDependencyNoProtectedReference,
    OperationDependencyUniqueValue,
]


# todo: switch to new syntax once pydantic supports it
OperationT = TypeVar('OperationT', bound=DatabaseOperation)


class SingleOperationWithDependencies(BaseSchema, Generic[OperationT]):
    uid: tuple
    operation: OperationT
    dependencies: set[OperationDependency] = set()

    @property
    def main_op(self) -> Self:
        return self


class MergableOperationsWithDependencies(BaseSchema):
    main_op: Union[
        SingleOperationWithDependencies[CreateObjectOperation],
        SingleOperationWithDependencies[UpdateObjectOperation],
    ]
    sub_ops: list[SingleOperationWithDependencies[UpdateObjectOperation]]

    @property
    def dependencies(self) -> set[OperationDependency]:
        return self.main_op.dependencies | reduce(operator.or_, (op.dependencies for op in self.sub_ops), set())


OperationWithDependencies = Union[
    SingleOperationWithDependencies,
    MergableOperationsWithDependencies,
]


class FoundObjectReference(BaseSchema):
    model_config = ConfigDict(frozen=True)

    obj: ObjectReference
    field: FieldName
    on_delete: str


class DummyValue:
    pass


class OperationSituation(BaseSchema):
    # operations done so far
    operations: list[DatabaseOperation] = []

    # uids of operationswithdependencies that are included now
    operation_uids: frozenset[tuple] = frozenset()

    # remaining operations still to do
    remaining_operations_with_dependencies: list[OperationWithDependencies] = []

    # objects that still need to be created before some remaining operation (or that were simply deleted in this run)
    missing_objects: dict[ModelName, set[ObjectID]] = {}

    # unique values relevant for these operations that are currently not free
    occupied_unique_values: dict[ModelName, dict[FieldName, dict[Any, ObjectID]]] = {}

    # references to objects that need to be removed for in this run
    obj_references: dict[ModelName, dict[ObjectID, set[FoundObjectReference]]] = {}

    def fulfils_dependency(self, dependency: OperationDependency) -> bool:
        if isinstance(dependency, OperationDependencyObjectExists):
            return dependency.obj.id not in self.missing_objects.get(dependency.obj.model, set())

        if isinstance(dependency, OperationDependencyNoProtectedReference):
            return not any(
                (reference.on_delete == "PROTECT") for reference in
                self.obj_references.get(dependency.obj.model, {}).get(dependency.obj.id, ())
            )

        if isinstance(dependency, OperationDependencyUniqueValue):
            return dependency.value not in self.occupied_unique_values.get(dependency.model, {}).get(
                dependency.field, set()
            )

        raise ValueError

    def fulfils_dependencies(self, dependencies: set[OperationDependency]) -> bool:
        return all(self.fulfils_dependency(dependency) for dependency in dependencies)


class ChangedObjectProblems(BaseSchema):
    obj_does_not_exist: bool = False
    cant_create: bool = False
    protected_references: set[FoundObjectReference] = set()
    field_does_not_exist: set[FieldName] = set()
    m2m_val_does_not_exist: dict[FieldName, set[ObjectID]] = {}
    dummy_values: dict[FieldName, Any] = {}
    ref_doesnt_exist: set[FieldName] = set()
    unique_constraint: set[FieldName] = set()

    def clean(self) -> bool:
        """
        Clean up data and return true if there's any problemls left
        """
        self.m2m_val_does_not_exist = {
            field_name: ids
            for field_name, ids in self.m2m_val_does_not_exist.items()
            if ids
        }
        return (
            self.obj_does_not_exist
            or self.cant_create
            or self.protected_references
            or self.field_does_not_exist
            or self.m2m_val_does_not_exist
            or self.dummy_values
            or self.ref_doesnt_exist
            or self.unique_constraint
        )


class ChangeProblems(BaseSchema):
    model_does_not_exist: set[ModelName] = set()
    objects: dict[ModelName, dict[ObjectID, ChangedObjectProblems]] = {}

    def clean(self):
        self.objects = {
            model_name: problem_objects for
            model_name, problem_objects in (
                (model_name, {pk: obj for pk, obj in problem_objects.items() if obj.clean()})
                for model_name, problem_objects in self.objects.items()
            )
            if problem_objects
        }

    def get_object(self, obj: ObjectReference):
        return self.objects.setdefault(obj.model, {}).setdefault(obj.id, ChangedObjectProblems())

    @property
    def any(self) -> bool:
        """ Are there any problems? """
        return bool(self.model_does_not_exist or self.objects)


# todo: what if model does not exist…


class ChangedObjectCollection(BaseSchema):
    """
    A collection of ChangedObject instances, sorted by model and id.
    Also stores a PreviousObjectCollection for comparison with the current state.
    Iterable as a list of ChangedObject instances.
    """
    prev: PreviousObjectCollection = PreviousObjectCollection()
    objects: dict[ModelName, dict[ObjectID, ChangedObject]] = {}

    def __iter__(self):
        yield from chain(*(objects.values() for model, objects in self.objects.items()))

    def __len__(self):
        return sum(len(v) for v in self.objects.values())

    def add_operations(self, operations: DatabaseOperationCollection):
        """
        Add the given operations, creating/updating changed objects to represent the resulting state.
        """
        # todo: if something is being changed back, remove it from thingy?
        self.prev.add_other(operations.prev)
        for operation in operations:
            changed_object = self.objects.setdefault(operation.obj.model, {}).get(operation.obj.id, None)
            if changed_object is None:
                # todo: titles should be better, probably
                titles = (
                    operation.fields.get("title", {})
                    if isinstance(operation, CreateObjectOperation)
                    else self.prev.get(operation.obj).titles
                )
                changed_object = ChangedObject(obj=operation.obj, titles=titles)
                self.objects[operation.obj.model][operation.obj.id] = changed_object
            if isinstance(operation, CreateObjectOperation):
                changed_object.created = True
                changed_object.fields.update(operation.fields)
            elif isinstance(operation, UpdateObjectOperation):
                model = apps.get_model('mapdata', operation.obj.model)
                for field_name, value in operation.fields.items():
                    field = model._meta.get_field(field_name)
                    if isinstance(field, I18nField) and field_name in changed_object.fields:
                        changed_object.fields[field_name] = {
                            lang: val for lang, val in {**changed_object.fields[field_name], **value}.items()
                        }
                    else:
                        changed_object.fields[field_name] = value
            elif isinstance(operation, DeleteObjectOperation):
                changed_object.deleted = True
            else:
                changed_m2m = changed_object.m2m_changes.get(operation.field, None)
                if changed_m2m is None:
                    changed_m2m = ChangedManyToMany()
                    changed_object.m2m_changes[operation.field] = changed_m2m
                if isinstance(operation, ClearManyToManyOperation):
                    changed_m2m.cleared = True
                    changed_m2m.added = []
                    changed_m2m.removed = []
                else:
                    changed_m2m.added = sorted((set(changed_m2m.added) | operation.add_values)
                                               - operation.remove_values)
                    changed_m2m.removed = sorted((set(changed_m2m.removed) - operation.add_values)
                                                 | operation.remove_values)

    def clean_and_complete_prev(self):
        # todo: what the heck was this function for?
        ids: dict[ModelName, set[ObjectID]] = {}
        for model_name, changed_objects in self.objects.items():
            ids.setdefault(model_name, set()).update(set(changed_objects.keys()))
            model = apps.get_model("mapdata", model_name)
            relations: dict[FieldName, Type[Model]] = {field.name: field.related_model
                                                       for field in model.get_fields() if field.is_relation}
            for obj in changed_objects.values():
                for field_name, value in obj.fields.items():
                    related_model = relations.get(field_name, None)
                    if related_model is None or value is None:
                        continue
                    ids.setdefault(related_model._meta.model_name, set()).add(value)
                for field_name, field_changes in obj.m2m_changes.items():
                    related_model = relations[field_name]
                    if field_changes.added or field_changes.removed:
                        ids.setdefault(related_model._meta.model_name, set()).update(field_changes.added)
                        ids.setdefault(related_model._meta.model_name, set()).update(field_changes.removed)
        # todo: move this to some kind of "usage explanation" function, implement rest of this

    class OperationsWithDependencies(NamedTuple):
        obj_operations: list[OperationWithDependencies]
        m2m_operations: list[SingleOperationWithDependencies]

    @property
    def as_operations_with_dependencies(self) -> tuple[OperationsWithDependencies, ChangeProblems]:
        operations_with_dependencies = self.OperationsWithDependencies(obj_operations=[], m2m_operations=[])
        problems = ChangeProblems()
        for model_name, changed_objects in self.objects.items():
            try:
                model = apps.get_model("mapdata", model_name)
            except LookupError:
                problems.model_does_not_exist.add(model_name)
                continue

            for changed_obj in changed_objects.values():
                base_dependencies: set[OperationDependency] = (
                    set() if changed_obj.created else {OperationDependencyObjectExists(obj=changed_obj.obj)}
                )

                if changed_obj.deleted:
                    if changed_obj.created:
                        continue
                    operations_with_dependencies.obj_operations.append(
                        SingleOperationWithDependencies(
                            uid=(changed_obj.obj, "delete"),
                            operation=DeleteObjectOperation(obj=changed_obj.obj),
                            dependencies=(
                                base_dependencies | {OperationDependencyNoProtectedReference(obj=changed_obj.obj)}
                            ),
                        ),
                    )
                    continue

                initial_fields = dict()
                obj_sub_operations: list[OperationWithDependencies] = []
                for field_name, value in changed_obj.fields.items():
                    try:
                        field = model._meta.get_field(field_name)
                    except FieldDoesNotExist:
                        problems.get_object(changed_obj.obj).field_does_not_exist.add(field_name)
                        continue

                    if value is None:
                        initial_fields[field_name] = None
                        continue
                    dependencies = base_dependencies.copy()
                    # todo: prev
                    if field.is_relation and not field.many_to_many:
                        dependencies.add(OperationDependencyObjectExists(obj=ObjectReference(
                            model=field.related_model._meta.model_name,
                            id=value,
                        )))
                    if field.unique:
                        dependencies.add(OperationDependencyUniqueValue(
                            model=model._meta.model_name,
                            field=field_name,
                            value=value,
                        ))

                    if not dependencies:
                        initial_fields[field_name] = value
                        continue

                    initial_fields[field_name] = None if field.null else DummyValue
                    obj_sub_operations.append(SingleOperationWithDependencies(
                        uid=(changed_obj.obj, f"field_{field_name}"),
                        operation=UpdateObjectOperation(obj=changed_obj.obj, fields={field_name: value}),
                        dependencies=dependencies
                    ))

                obj_main_operation = SingleOperationWithDependencies(
                    uid=(changed_obj.obj, f"main"),
                    operation=(CreateObjectOperation if changed_obj.created else UpdateObjectOperation)(
                        obj=changed_obj.obj,
                        fields=initial_fields,
                    ),
                    dependencies=base_dependencies,
                )

                if not obj_sub_operations:
                    operations_with_dependencies.obj_operations.append(obj_main_operation)
                else:
                    operations_with_dependencies.obj_operations.append(MergableOperationsWithDependencies(
                        main_op=obj_main_operation,
                        sub_ops=obj_sub_operations,
                    ))

                for field_name, m2m_changes in changed_obj.m2m_changes.items():
                    if m2m_changes.cleared:
                        operations_with_dependencies.m2m_operations.append(SingleOperationWithDependencies(
                            uid=(changed_obj.obj, f"m2mclear_{field_name}"),
                            operation=ClearManyToManyOperation(
                                obj=changed_obj.obj,
                                field=field_name,
                            ),
                            dependencies={OperationDependencyObjectExists(obj=changed_obj.obj)},
                        ))
                    if m2m_changes.added or m2m_changes.removed:
                        operations_with_dependencies.m2m_operations.append(SingleOperationWithDependencies(
                            uid=(changed_obj.obj, f"m2mupdate_{field_name}"),
                            operation=UpdateManyToManyOperation(
                                obj=changed_obj.obj,
                                field=field_name,
                                add_values=m2m_changes.added,
                                remove_values=m2m_changes.removed,
                            ),
                            dependencies={OperationDependencyObjectExists(obj=changed_obj.obj)},
                        ))

        return operations_with_dependencies, problems

    class CreateStartOperationResult(NamedTuple):
        situation: OperationSituation
        unique_values_needed: dict[ModelName, dict[FieldName: set]]
        m2m_operations: list[SingleOperationWithDependencies]
        problems: ChangeProblems

    def create_start_operation_situation(self) -> CreateStartOperationResult:
        operations_with_dependencies, problems = self.as_operations_with_dependencies

        start_situation = OperationSituation(
            remaining_operations_with_dependencies=operations_with_dependencies.obj_operations
        )

        referenced_objects: dict[ModelName, set[ObjectID]] = {}  # objects that need to exist before
        deleted_existing_objects: dict[ModelName, set[ObjectID]] = {}  # objects that need to exist before
        unique_values_needed: dict[ModelName, dict[FieldName: set]] = {}
        for operation in operations_with_dependencies.obj_operations:
            for dependency in operation.dependencies:
                if isinstance(dependency, OperationDependencyObjectExists):
                    referenced_objects.setdefault(dependency.obj.model, set()).add(dependency.obj.id)
                elif isinstance(dependency, OperationDependencyUniqueValue):
                    unique_values_needed.setdefault(dependency.model, {}).setdefault(
                        dependency.field, set()
                    ).add(dependency.value)
                elif isinstance(dependency, OperationDependencyNoProtectedReference):
                    deleted_existing_objects.setdefault(dependency.obj.model, set()).add(dependency.obj.id)

        # references from m2m changes need also to be checked if they exist
        for model_name, changed_objects in self.objects.items():
            try:
                model = apps.get_model("mapdata", model_name)
            except LookupError:
                # would already have been reported above
                continue
            # todo: how do we want m2m to work when it's cleared by the user but things were added in the meantime?
            for changed_obj in changed_objects.values():
                for field_name, m2m_changes in changed_obj.m2m_changes.items():
                    try:
                        field = model._meta.get_field(field_name)
                    except FieldDoesNotExist:
                        problems.get_object(changed_obj.obj).field_does_not_exist.add(field_name)
                        continue
                    referenced_objects.setdefault(
                        field.related_model._meta.model_name, set()
                    ).update(set(m2m_changes.added + m2m_changes.removed))

        # let's find which objects that need to exist before actually exist
        for model, ids in referenced_objects.items():
            model_cls = apps.get_model('mapdata', model)
            ids_found = set(model_cls.objects.filter(pk__in=ids).values_list('pk', flat=True))
            start_situation.missing_objects[model] = {id_ for id_ in ids if id_ not in ids_found}

        # let's find which unique values are actually occupied right now
        for model, fields in unique_values_needed.items():
            model_cls = apps.get_model('mapdata', model)
            q = Q()
            for field_name, values in fields.items():
                q |= Q(**{f'{field_name}__in': values})
            start_situation.occupied_unique_values[model] = {}
            for result in model_cls.objects.filter(q).values("id", *fields.keys()):
                pk = result.pop("id")
                for field_name, value in result.items():
                    if value in fields[field_name]:
                        start_situation.occupied_unique_values[model].setdefault(field_name, {})[value] = pk

        # let's find which protected references to objects we want to delete have
        potential_fields: dict[ModelName, dict[FieldName, dict[ModelName, set[ObjectID]]]] = {}
        for model, ids in deleted_existing_objects.items():
            # don't check this for objects that don't exist anymore
            ids -= start_situation.missing_objects.get(model, set())
            for field in apps.get_model('mapdata', model)._meta.get_fields():
                if (not isinstance(field, (ManyToOneRel, OneToOneRel))
                        or field.related_model._meta.app_label != "mapdata"):
                    continue
                potential_fields.setdefault(field.related_model._meta.model_name,
                                            {}).setdefault(field.field.attname, {})[model] = ids

        # collect all references to objects we want to delete
        for model, fields in potential_fields.items():
            model_cls = apps.get_model('mapdata', model)
            q = Q()
            targets_reverse: dict[FieldName, dict[ObjectID, ModelName]] = {}
            for field_name, targets in fields.items():
                ids = reduce(operator.or_, targets.values(), set())
                q |= Q(**{f'{field_name}__in': ids})
                targets_reverse[field_name] = dict(chain(*(((id_, target_model) for id_ in target_ids)
                                                           for target_model, target_ids in targets.items())))
            for result in model_cls.objects.filter(q).values("id", *fields.keys()):
                source_ref = ObjectReference(model=model, id=result.pop("id"))
                for field, target_id in result.items():
                    target_model = targets_reverse[field][target_id]
                    start_situation.obj_references.setdefault(target_model, {}).setdefault(target_id, set()).add(
                        FoundObjectReference(obj=source_ref, field=field,
                                             on_delete=model_cls._meta.get_field(field).remote_field.on_delete.__name__)
                    )

        return self.CreateStartOperationResult(
            situation=start_situation,
            unique_values_needed=unique_values_needed,
            m2m_operations=operations_with_dependencies.m2m_operations,
            problems=problems,
        )

    class ChangesAsOperations(NamedTuple):
        operations: DatabaseOperationCollection
        problems: ChangeProblems

    @property
    def as_operations(self) -> ChangesAsOperations:
        current_objects = {}
        for model_name, changed_objects in self.objects.items():
            model = apps.get_model("mapdata", model_name)
            current_objects[model_name] = {
                obj["pk"]: obj["fields"]
                for obj in json.loads(
                    serializers.serialize("json", model.objects.filter(pk__in=changed_objects.keys()))
                )
            }

        start_situation, unique_values_needed, m2m_operations, problems = self.create_start_operation_situation()

        # situations still to deal with, sorted by number of operations
        open_situations: list[OperationSituation] = [start_situation]

        # situation that solves for all operations
        done_situation: OperationSituation | None = None

        # situations that ended prematurely
        ended_situations: list[OperationSituation] = []

        # situations already encountered by set of operation uuids included, values are number of operations
        best_uids:  dict[frozenset[tuple], int] = {}

        # unique values in db [only want to check for them once]
        dummy_unique_value_avoid: dict[ModelName, dict[FieldName, frozenset]] = {}
        available_model_ids: dict[ModelName, frozenset] = {}

        if not start_situation.remaining_operations_with_dependencies:
            # nothing to do? then we're done
            done_situation = start_situation

        while open_situations and not done_situation:
            situation = open_situations.pop(0)

            continued = False
            for i, remaining_operation in enumerate(situation.remaining_operations_with_dependencies):
                # check if the main operation can be run
                if not situation.fulfils_dependencies(remaining_operation.main_op.dependencies):
                    continue

                # determine changes to state
                new_operation = remaining_operation.main_op.operation
                new_remaining_operations = []
                uids_to_add: set[tuple] = set(remaining_operation.main_op.uid)
                if isinstance(remaining_operation, MergableOperationsWithDependencies):
                    # sub_ops to be merged into this one or become pending operations
                    new_operation: Union[CreateObjectOperation, UpdateObjectOperation]
                    for sub_op in remaining_operation.sub_ops:
                        if situation.fulfils_dependencies(sub_op.dependencies):
                            new_operation.fields.update(sub_op.operation.fields)
                            uids_to_add.add(sub_op.uid)
                        else:
                            new_remaining_operations.append(sub_op)

                model_cls = apps.get_model('mapdata', new_operation.obj.model)
                if isinstance(new_operation, (CreateObjectOperation, UpdateObjectOperation)):
                    couldnt_fill_dummy = False
                    for field_name, value in tuple(new_operation.fields.items()):
                        if value is DummyValue:
                            # if there's a dummy value to fill, we need to find a dummy value
                            field = model_cls._meta.get_field(field_name)

                            if field.is_relation:
                                # for a relation, we will try to find a valid other object to reference
                                if available_model_ids.get(field.related_model._meta.model_name) is None:
                                    # find available model ids, we only query these once for less queries
                                    available_model_ids[field.related_model._meta.model_name] = frozenset(
                                        field.related_model.objects.values_list('pk', flat=True)
                                    )
                                if field.unique:
                                    # if the field is unique we need to fin a value that isn't occupied
                                    # and, to be sure, that we haven't used as a dummyvalue before
                                    if dummy_unique_value_avoid.get(new_operation.obj.model, {}).get(field_name) is None:
                                        dummy_unique_value_avoid.setdefault(
                                            new_operation.obj.model, {}
                                        )[field_name] = frozenset(
                                            model_cls.objects.values_list(field_name.attname, flat=True)
                                        ) | unique_values_needed.get(new_operation.obj.model, {}).get(field_name, set())

                                    choices = (
                                        available_model_ids[field.related_model._meta.model_name] -
                                        dummy_unique_value_avoid[new_operation.obj.model][field_name] -
                                        set(
                                            situation.occupied_unique_values[new_operation.obj.model][field_name].keys()
                                        )
                                    )
                                else:
                                    choices = available_model_ids[field.related_model._meta.model_name]
                                if not choices:
                                    couldnt_fill_dummy = True
                                    break
                                dummy_value = next(iter(choices))
                            else:
                                if field.unique:
                                    # otherwise, an non-relational field needs a unique value
                                    if dummy_unique_value_avoid.get(new_operation.obj.model, {}).get(field_name) is None:
                                        dummy_unique_value_avoid.setdefault(
                                            new_operation.obj.model, {}
                                        )[field_name] = frozenset(
                                            model_cls.objects.values_list(field_name, flat=True)
                                        ) | unique_values_needed.get(new_operation.obj.model, {}).get(field_name, set())
                                    occupied = (
                                        dummy_unique_value_avoid[new_operation.obj.model][field_name] -
                                        set(
                                            situation.occupied_unique_values[new_operation.obj.model][field_name].keys()
                                        )
                                    )
                                else:
                                    # this shouldn't happen, because dummy values are only used by non-relation fields
                                    # for unique constraints
                                    raise NotImplementedError

                                # generate a value that works
                                if isinstance(field, (SlugField, CharField)):
                                    new_val = "dummyvalue"
                                    while new_val in occupied:
                                        new_val = "dummyvalue"+str(random.randrange(1, 10000000))
                                elif isinstance(field, (DecimalField, IntegerField, SmallIntegerField,
                                                        PositiveIntegerField, PositiveSmallIntegerField)):
                                    new_val = 0
                                    while new_val in occupied:
                                        new_val += 1
                                else:
                                    raise NotImplementedError
                                dummy_value = new_val

                            # store the dummyvalue so we can tell the user about it
                            problems.get_object(new_operation.obj).dummy_values[field_name] = dummy_value

                        else:
                            # we have set this field to a non-dummy value, if we used one before we can forget it
                            problems.get_object(new_operation.obj).dummy_values.pop(field_name, None)

                    if couldnt_fill_dummy:
                        continue  # if we couldn't fill a dummy value this operation is not doable, skip it

                # construct new situation
                new_situation = situation.model_copy(deep=True)

                if isinstance(new_operation, CreateObjectOperation) and new_situation.operations:
                    last_operation = new_situation.operations[-1]
                    if (isinstance(last_operation, CreateObjectOperation) and
                            last_operation.obj.model == new_operation.obj.model):
                        new_situation.operations[-1] = CreateMultipleObjectsOperation(
                            objects=[last_operation, new_operation],
                        )
                    elif (isinstance(last_operation, CreateMultipleObjectsOperation) and
                            last_operation.objects[-1].obj.model == new_operation.obj.model):
                        last_operation.objects.append(new_operation)
                    else:
                        new_situation.operations.append(new_operation)
                else:
                    if not (isinstance(new_operation, UpdateObjectOperation) and not new_operation.fields):
                        # we might have empty update operations, those can be ignored
                        new_situation.operations.append(new_operation)

                new_situation.remaining_operations_with_dependencies.pop(i)
                new_situation.remaining_operations_with_dependencies.extend(new_remaining_operations)
                new_situation.operation_uids = new_situation.operation_uids | uids_to_add

                # even if we don't actually continue cause better paths existed, this situation is not a deadlock
                continued = True

                if not new_situation.remaining_operations_with_dependencies:
                    # nothing left to do, congratulations we did it!
                    done_situation = new_situation
                    break

                if best_uids.get(new_situation.operation_uids, 1000000) <= len(new_situation.operations):
                    # we already reached this situation with the same or less amount of operations
                    continue

                if isinstance(new_operation, CreateObjectOperation):
                    # if an object was created it's no longer missing
                    new_situation.missing_objects.get(new_operation.obj.model, set()).discard(new_operation.obj.id)

                if isinstance(new_operation, UpdateObjectOperation):
                    occupied_unique_values = new_situation.occupied_unique_values.get(new_operation.obj.model, {})
                    relations_changed = set()
                    for field_name in new_operation.fields:
                        field = model_cls._meta.get_field(field_name)
                        if field.unique:
                            # unique field was changed? remove unique value entry [might be readded below]
                            occupied_unique_values[field_name] = {
                                val: pk for val, pk in occupied_unique_values.get(field_name, {}).items()
                                if pk != new_operation.obj.model
                            }
                        if field.is_relation:
                            relations_changed.add(field_name)

                    if relations_changed:
                        # relation field was changed? remove reference entry [might be readded below]
                        for model_name, references in tuple(new_situation.obj_references.items()):
                            new_situation.obj_references[model_name] = {
                                pk: ref for pk, ref in references.items()
                                if ref.obj != new_operation.obj or ref.field not in relations_changed
                            }

                if isinstance(new_operation, DeleteObjectOperation):
                    # if an object was deleted it will now be missing
                    new_situation.missing_objects.get(new_operation.obj.model, set()).add(new_operation.obj.id)

                    # all unique values it occupied will no longer be occupied
                    occupied_unique_values = new_situation.occupied_unique_values.get(new_operation.obj.model, {})
                    for field_name, values in tuple(occupied_unique_values.items()):
                        occupied_unique_values[field_name] = {val: pk for val, pk in values.items()
                                                              if pk != new_operation.obj.model}

                    # all references that came from it, will no longer exist
                    for model_name, references in tuple(new_situation.obj_references.items()):
                        new_situation.obj_references[model_name] = {
                            pk: {ref for ref in refs if ref.obj != new_operation.obj}
                            for pk, refs in references.items()
                        }

                    # todo: we ignore cascading for now, do we want to keep it that way?
                else:
                    for field_name, value in new_operation.fields.items():
                        field = model_cls._meta.get_field(field_name)
                        if value is None:
                            continue
                        if field.unique:
                            # unique field was changed? add unique value entry
                            new_situation.occupied_unique_values.setdefault(
                                new_operation.obj.model, {}
                            ).setdefault(field_name, {})[value] = new_operation.obj.id
                        if field.is_relation and not field.many_to_many:
                            # relation field was changed? add foundobjectreference
                            new_situation.obj_references.setdefault(
                                field.related_model._meta.model_name, {}
                            ).setdefault(value, set()).add(
                                FoundObjectReference(
                                    obj=new_operation.obj,
                                    field=field_name,
                                    on_delete=field.remote_field.on_delete.__name__,
                                )
                            )

                # finally insert new situation
                bisect.insort(open_situations, new_situation, key=lambda s: len(s.operations))
                best_uids[new_situation.operation_uids] = len(new_situation.operations)

            if not continued:
                ended_situations.append(situation)

        if not done_situation:
            done_situation = max(ended_situations, key=lambda s: (len(s.operation_uuids), -len(s.operations)))

        # add m2m
        for m2m_operation_with_dependencies in m2m_operations:
            if not done_situation.fulfils_dependencies(m2m_operation_with_dependencies.dependencies):
                done_situation.remaining_operations_with_dependencies.append(m2m_operation_with_dependencies)
                continue
            done_situation.operations.append(m2m_operation_with_dependencies.operation)

        for remaining_operation in done_situation.remaining_operations_with_dependencies:
            model_cls = apps.get_model("mapdata", remaining_operation.main_op.obj.model)
            obj = remaining_operation.main_op.obj
            problem_obj = problems.get_object(obj)
            if obj.id in done_situation.missing_objects.get(obj.model, set()):
                problem_obj.obj_does_not_exist = True
                continue

            if isinstance(remaining_operation.main_op, DeleteObjectOperation):
                problem_obj.protected_references = {
                    found_ref for found_ref in done_situation.obj_references.get(
                        remaining_operation.main_op.obj.model, {}
                    )[remaining_operation.main_op.obj.id]
                    if found_ref.on_delete == "PROTECT"
                }
                # this will fail if there are no protected references because that should never happen
                continue

            if isinstance(remaining_operation.main_op, CreateObjectOperation):
                problem_obj.cant_create = True

            sub_ops = (chain((remaining_operation.main_op,), remaining_operation.sub_ops)
                       if isinstance(remaining_operation, MergableOperationsWithDependencies)
                       else (remaining_operation.main_op,))
            for sub_op in sub_ops:
                if isinstance(sub_op, UpdateManyToManyOperation):
                    related_model_name = model_cls._meta.get_field(sub_op.field).related_model._meta.model_name
                    missing_ids = (
                        done_situation.missing_objects.get(related_model_name, set()) &
                        (sub_op.add_values | sub_op.remove_values)
                    )
                    if missing_ids:
                        problem_obj[sub_op.field] = missing_ids
                elif isinstance(sub_op, UpdateObjectOperation):
                    for dependency in sub_op.dependencies:
                        if isinstance(dependency, OperationDependencyObjectExists) and dependency.obj != sub_op.obj:
                            problem_obj.ref_doesnt_exist.update(set(sub_op.fields.keys()))
                        elif isinstance(dependency, OperationDependencyUniqueValue):
                            problem_obj.unique_constraint.update(set(sub_op.fields.keys()))

        problems.clean()

        return self.ChangesAsOperations(
            operations=DatabaseOperationCollection(
                prev=self.prev,
                operations=done_situation.operations,
            ),
            problems=problems
        )