import ast
import functools
import json
import logging
import re
import traceback
from collections import defaultdict
from copy import deepcopy
from uuid import uuid4

import httpx
from core.choices import JobStatusChoices
from core.exceptions import SyncError
from core.models import Job
from core.models import ObjectType
from core.signals import pre_sync
from dcim.models import MACAddress
from dcim.models import Site
from django.apps import apps
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator
from django.db import models
from django.db import transaction
from django.db.models import Q
from django.db.models import QuerySet
from django.db.models import signals
from django.db.models.signals import m2m_changed
from django.dispatch import receiver
from django.urls import reverse
from django.utils import timezone
from django.utils.module_loading import import_string
from django.utils.translation import gettext as _
from netbox.context import current_request
from netbox.models import ChangeLoggedModel
from netbox.models import NetBoxModel
from netbox.models import PrimaryModel
from netbox.models.features import JobsMixin
from netbox.models.features import TagsMixin
from netbox_branching.choices import BranchStatusChoices
from netbox_branching.contextvars import active_branch
from netbox_branching.models import Branch
from netbox_branching.utilities import supports_branching
from rq.timeouts import JobTimeoutException
from utilities.querysets import RestrictedQuerySet
from utilities.request import NetBoxFakeRequest

from . import NetboxIPFabricConfig
from .choices import IPFabricEndpointChoices
from .choices import IPFabricFilterTypeChoices
from .choices import IPFabricRawDataTypeChoices
from .choices import IPFabricSnapshotStatusModelChoices
from .choices import IPFabricSourceStatusChoices
from .choices import IPFabricSourceTypeChoices
from .choices import IPFabricSyncParameterChoices
from .choices import IPFabricSyncStatusChoices
from .choices import required_transform_map_contenttypes
from .signals import assign_primary_mac_address
from .utilities.ipfutils import IPFabric
from .utilities.ipfutils import IPFabricSyncRunner
from .utilities.ipfutils import render_jinja2
from .utilities.logging import SyncLogging
from .utilities.transform_map import has_cycle_dfs

logger = logging.getLogger("ipfabric_netbox.models")


def apply_tags(object, tags, connection_name=None):
    def _apply(object):
        object.snapshot()
        for tag in tags:
            if hasattr(object, "tags"):
                object.tags.add(tag)
        object.save(using=connection_name)

    _apply(object)


IPFabricSupportedSyncModels = Q(
    Q(app_label="dcim", model="site")
    | Q(app_label="dcim", model="manufacturer")
    | Q(app_label="dcim", model="platform")
    | Q(app_label="dcim", model="devicerole")
    | Q(app_label="dcim", model="devicetype")
    | Q(app_label="dcim", model="device")
    | Q(app_label="dcim", model="virtualchassis")
    | Q(app_label="dcim", model="interface")
    | Q(app_label="dcim", model="macaddress")
    | Q(app_label="ipam", model="vlan")
    | Q(app_label="ipam", model="vrf")
    | Q(app_label="ipam", model="prefix")
    | Q(app_label="ipam", model="ipaddress")
    | Q(app_label="contenttypes", model="contenttype")
    | Q(app_label="tenancy", model="tenant")
    | Q(app_label="dcim", model="inventoryitem")
)


IPFabricRelationshipFieldSourceModels = Q(
    Q(app_label="dcim")
    | Q(app_label="ipam")
    | Q(app_label="tenancy")
    | Q(app_label="contenttypes", model="contenttype")
)


class IPFabricEndpoint(NetBoxModel):
    objects = RestrictedQuerySet.as_manager()

    name = models.CharField(max_length=100, unique=True)
    description = models.TextField(blank=True, null=True)
    endpoint = models.CharField(
        max_length=200,
        verbose_name=_(
            "Endpoint path from URL notation, for example `/inventory/devices`."
        ),
        choices=IPFabricEndpointChoices,
        unique=True,
    )

    class Meta:
        ordering = ("pk",)
        verbose_name = _("IP Fabric Endpoint")
        verbose_name_plural = _("IP Fabric Endpoints")

    def __str__(self):
        return f"{self.endpoint}"

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricendpoint", args=[self.pk])

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        if not self.endpoint.startswith("/"):
            self.endpoint = f"/{self.endpoint}"
        if self.endpoint.endswith("/"):
            self.endpoint = self.endpoint.rstrip("/")

    @staticmethod
    def _merge_filter_structures(base: dict, new: dict) -> dict:
        """Recursively merge filter structures with matching and/or keys at same level.

        Args:
            base: Base filter dictionary to merge into
            new: New filter dictionary to merge from

        Returns:
            Merged filter dictionary
        """
        for key, value in new.items():
            # Only merge 'and' and 'or' keys
            if key not in ("and", "or") or not isinstance(value, list):
                continue

            if key not in base:
                base[key] = []

            # Process each item in the new filter's array
            for new_item in value:
                if not isinstance(new_item, dict):
                    # Non-dict items just get appended
                    base[key].append(new_item)
                    continue

                # Check if there's a matching structure in base to merge with
                merged = False
                for base_item in base[key]:
                    if not isinstance(base_item, dict):
                        continue

                    # Check if both dicts have the same and/or keys
                    new_keys = set(k for k in new_item.keys() if k in ("and", "or"))
                    base_keys = set(k for k in base_item.keys() if k in ("and", "or"))

                    if new_keys == base_keys and new_keys:
                        # Matching structure found - recursively merge
                        IPFabricEndpoint._merge_filter_structures(base_item, new_item)
                        merged = True
                        break

                if not merged:
                    # No matching structure found - append as new item
                    base[key].append(new_item)

        return base

    def combine_filters(self, sync=None) -> dict:
        """Combine all filters for this endpoint into a single filter dictionary.

        Args:
            sync: Optional IPFabricSync to filter by. If provided, only filters
                  associated with that sync are included.

        Returns:
            Dict with filter types as keys (e.g., 'and', 'or') and lists of
            expressions as values.
        """
        combined_filter = {}

        # Get filters for this endpoint, optionally filtered by sync
        if sync:
            endpoint_filters = self.filters.filter(syncs=sync)
        else:
            endpoint_filters = self.filters.all()

        for endpoint_filter in endpoint_filters:
            filter_expressions = endpoint_filter.merge_expressions()

            # Create a temporary dict with the filter type as key
            new_filter = {endpoint_filter.filter_type: filter_expressions}

            # Recursively merge the new filter into combined_filter
            combined_filter = self._merge_filter_structures(combined_filter, new_filter)

        # Sites filter is stored in sync parameters for user convenience
        if sync and (sites := (sync.parameters or {}).get("sites")):
            if "and" not in combined_filter:
                combined_filter["and"] = []
            combined_filter["and"].extend(
                [{"or": [{"siteName": ["eq", site]} for site in sites]}]
            )

        return combined_filter


class IPFabricTransformMapGroup(NetBoxModel):
    name = models.CharField(max_length=100, unique=True)
    description = models.TextField(blank=True, null=True)

    class Meta:
        ordering = ("pk",)
        verbose_name = _("IP Fabric Transform Map Group")
        verbose_name_plural = _("IP Fabric Transform Map Groups")

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse(
            "plugins:ipfabric_netbox:ipfabrictransformmapgroup", args=[self.pk]
        )


class IPFabricTransformMap(NetBoxModel):
    name = models.CharField(max_length=200)
    source_endpoint = models.ForeignKey(
        to=IPFabricEndpoint,
        on_delete=models.PROTECT,
        related_name="transform_maps",
        editable=True,
    )
    target_model = models.ForeignKey(
        to=ContentType,
        related_name="+",
        verbose_name=_("Target Model"),
        limit_choices_to=IPFabricSupportedSyncModels,
        help_text=_("The object(s) to which transform map target applies."),
        on_delete=models.PROTECT,
        blank=False,
        null=False,
    )
    group = models.ForeignKey(
        to=IPFabricTransformMapGroup,
        on_delete=models.CASCADE,
        related_name="transform_maps",
        blank=True,
        null=True,
    )
    parents = models.ManyToManyField(
        "self",
        symmetrical=False,
        blank=True,
        related_name="children",
        help_text=_(
            "Parent transform maps, for hierarchical organization during sync."
        ),
    )

    class Meta:
        ordering = ("pk",)
        verbose_name = _("IP Fabric Transform Map")
        verbose_name_plural = _("IP Fabric Transform Maps")

    def __str__(self):
        try:
            if self.source_endpoint and self.target_model:
                return f"{self.source_endpoint} - {self.target_model}"
        except (AttributeError, IPFabricEndpoint.DoesNotExist):
            pass
        return f"Transform Map: {self.name}" if self.name else "Transform Map"

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabrictransformmap", args=[self.pk])

    @property
    def docs_url(self):
        # TODO: Add docs url
        return ""

    def clean(self):
        cleaned_data = super().clean()
        qs = IPFabricTransformMap.objects.filter(
            group=self.group,
            target_model_id=self.target_model_id,
            source_endpoint_id=self.source_endpoint_id,
        )
        if self.pk:
            qs = qs.exclude(pk=self.pk)
        if qs.exists():
            err_msg = _(
                f"A transform map with group '{self.group}', target model '{self.target_model}', and source endpoint '{self.source_endpoint}' already exists."
            )
            raise ValidationError(
                {
                    "group": err_msg,
                    "target_model": err_msg,
                    "source_endpoint": err_msg,
                }
            )

        # Validate no circular dependencies (only if saved and has parents)
        if self.pk:
            self._validate_no_circular_dependency()

        return cleaned_data

    def _validate_no_circular_dependency(self):
        """
        Check if the current parent relationships create a circular dependency.
        Uses DFS to detect cycles in the directed graph.
        """

        def get_parents(node_id: int, parent_override: list | None) -> models.QuerySet:
            """Get parents for a node."""
            node = IPFabricTransformMap.objects.get(pk=node_id)
            return node.parents.all()

        if has_cycle_dfs(self.pk, get_parents):
            raise ValidationError(
                {
                    "parents": _(
                        "The selected parents create a circular dependency. "
                        "A transform map cannot be an ancestor of itself."
                    )
                }
            )

    @functools.cache
    def get_all_models(self):
        _context = dict()

        for app, app_models in apps.all_models.items():
            _context.setdefault(app, {})
            for model in app_models:
                if isinstance(model, str):
                    model = apps.get_registered_model(app, model)
                if not supports_branching(model):
                    continue
                _context[app][model.__name__] = model
        _context["contenttypes"] = {}
        _context["contenttypes"]["ContentType"] = ContentType
        return _context

    @classmethod
    def get_distinct_target_models(cls) -> QuerySet[ContentType]:
        target_model_ids = IPFabricTransformMap.objects.values_list(
            "target_model", flat=True
        ).distinct()
        return ContentType.objects.filter(id__in=target_model_ids)

    def build_relationships(self, source_data, connection_name: str | None):
        relationship_maps = self.relationship_maps.all()
        rel_dict = {}
        rel_dict_coalesce = {}

        for field in relationship_maps:
            if not field.template:
                continue
            context = {
                "object": source_data,
            }
            context.update(self.get_all_models())
            text = render_jinja2(
                field.template.replace(
                    ".objects.", f".objects.using('{connection_name}')."
                ),
                context,
            ).strip()
            if text:
                try:
                    pk = int(text)
                except ValueError:
                    pk = text

                if isinstance(pk, int):
                    related_object = (
                        field.source_model.model_class()
                        .objects.using(connection_name)
                        .get(pk=pk)
                    )
                else:
                    related_object = ast.literal_eval(pk)

                if not field.coalesce:
                    rel_dict[field.target_field] = related_object
                else:
                    if related_object is None:
                        # We are searching by this field, so we need to set it to None
                        rel_dict_coalesce[field.target_field + "__isnull"] = True
                    else:
                        rel_dict_coalesce[field.target_field] = related_object
        return rel_dict, rel_dict_coalesce

    def strip_source_data(self, source_data: dict) -> dict:
        """Strip data according to Transform Map mappings but without rendering templates."""
        keys = set()
        for field in self.field_maps.all():
            keys.add(field.source_field)
            if field.template:
                keys.update(
                    re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
                )
        for field in self.relationship_maps.all():
            if field.template:
                keys.update(
                    re.findall(r"object\.([a-zA-Z_0-9]+)(?=.*)", field.template)
                )
        # FIXME: Make it raise KeyError when key is missing during IN-68
        #  This is temporary hack to allow missing keys when syncing VSS
        return {k: source_data.get(k) for k in keys}

    def get_context(self, source_data, connection_name: str | None = None):
        new_data = deepcopy(source_data)
        relationship, coalesce_relationship = self.build_relationships(
            source_data=source_data,
            connection_name=connection_name,
        )
        if relationship:
            new_data["relationship"] = relationship
        if coalesce_relationship:
            new_data["relationship_coalesce"] = coalesce_relationship
        context = self.render(new_data, connection_name)
        return context

    def update_or_create_instance(self, context, tags=None, connection_name=None):
        tags = tags or []
        target_class = self.target_model.model_class()
        queryset = target_class.objects.using(connection_name)

        # Don't change context since it's used in case of exception for IPFabricIngestionIssue
        context = deepcopy(context)
        defaults = context.pop("defaults", {})

        with transaction.atomic(using=connection_name):
            try:
                # For correct ObjectChange on UPDATE we need to create snapshot
                # NetBox does this in UI using views, we need to do it manually
                # See NetBox docs Customization -> Custom Scripts -> Change Logging
                instance = queryset.get(**context)
                changed = False
                for attr, value in defaults.items():
                    # Only run data validation and save if something has changed
                    if getattr(instance, attr) == value:
                        continue
                    if not changed:
                        instance.snapshot()
                    changed = True
                    setattr(instance, attr, value)
                if changed:
                    instance.full_clean()
                    instance.save(using=connection_name)
            except target_class.DoesNotExist:
                for field in list(context.keys()):
                    # When assigning we need to replace `field__isnull=True` with `field=None`
                    if field.endswith("__isnull"):
                        context[field[:-8]] = None
                        del context[field]
                # Using queryset.create() creates the object even when it fails on clean()
                # To to work around it, we do it in two steps to avoid saving it to DB before clean()
                instance = queryset.model(**context, **defaults)
                instance.full_clean()
                instance.save(using=connection_name)

            apply_tags(instance, tags, connection_name)

        return instance

    def render(self, source_data, connection_name: str | None):
        data = {"defaults": {}}
        for field in self.field_maps.all():
            if field.template:
                context = {
                    "object": source_data,
                    field.source_field: source_data[field.source_field],
                }
                context.update(self.get_all_models())
                text = render_jinja2(
                    field.template.replace(
                        ".objects.", f".objects.using('{connection_name}')."
                    ),
                    context,
                ).strip()
            else:
                text = source_data[field.source_field]

            if text is not None:
                if isinstance(text, str):
                    if text.lower() in ["true"]:
                        text = True
                    elif text.lower() in ["false"]:
                        text = False
                    elif text.lower() in ["none"]:
                        text = None

                    if text:
                        target_field = getattr(
                            self.target_model.model_class(), field.target_field
                        )
                        target_field_type = target_field.field.get_internal_type()
                        if "integer" in target_field_type.lower():
                            text = int(text)

            if not field.coalesce:
                data["defaults"][field.target_field] = text
            else:
                if text is None:
                    data[field.target_field + "__isnull"] = True
                else:
                    data[field.target_field] = text

        if relationship := source_data.get("relationship"):
            data["defaults"].update(relationship)

        if relationship_coalesce := source_data.get("relationship_coalesce"):
            data.update(relationship_coalesce)

        return data


class IPFabricRelationshipField(models.Model):
    transform_map = models.ForeignKey(
        to=IPFabricTransformMap,
        on_delete=models.CASCADE,
        related_name="relationship_maps",
        editable=True,
    )
    source_model = models.ForeignKey(
        ContentType,
        related_name="ipfabric_transform_fields",
        limit_choices_to=IPFabricRelationshipFieldSourceModels,
        verbose_name=_("Source Model"),
        on_delete=models.PROTECT,
        blank=False,
        null=False,
    )
    target_field = models.CharField(max_length=100)
    coalesce = models.BooleanField(default=False)
    template = models.TextField(
        help_text=_(
            "Jinja2 template code, return an integer to create a relationship between the source and target model. True, False and None are also supported."
        ),
        blank=True,
        default="",
    )

    objects = RestrictedQuerySet.as_manager()

    class Meta:
        ordering = ("transform_map",)
        verbose_name = _("IP Fabric Relationship Field")
        verbose_name_plural = _("IP Fabric Relationship Fields")

    @property
    def docs_url(self):
        # TODO: Add docs url
        return ""


class IPFabricTransformField(models.Model):
    transform_map = models.ForeignKey(
        to=IPFabricTransformMap,
        on_delete=models.CASCADE,
        related_name="field_maps",
        editable=True,
    )
    source_field = models.CharField(max_length=100)
    target_field = models.CharField(max_length=100)
    coalesce = models.BooleanField(default=False)

    objects = RestrictedQuerySet.as_manager()

    template = models.TextField(
        help_text=_("Jinja2 template code to be rendered into the target field."),
        blank=True,
        default="",
    )

    class Meta:
        ordering = ("transform_map",)
        verbose_name = _("IP Fabric Transform Field")
        verbose_name_plural = _("IP Fabric Transform Fields")

    @property
    def docs_url(self):
        # TODO: Add docs url
        return ""


class IPFabricClient:
    def get_client(self, parameters):
        try:
            ipf = IPFabric(parameters=parameters)
            return ipf.ipf
        except httpx.ConnectError as e:
            if "CERTIFICATE_VERIFY_FAILED" in str(e):
                error_message = (
                    "SSL certificate verification failed, self-signed cert? "
                    "<a href='https://docs.ipfabric.io/main/integrations/netbox-plugin/user_guide/10_FAQ/' target='_blank'>Check out our FAQ documentation.</a>"
                )
            else:
                error_message = str(e)
            self.handle_sync_failure("ConnectError", e, error_message)
        except httpx.HTTPStatusError as e:
            if e.response.status_code == 401:
                error_message = "Authentication failed, check API key."
            else:
                error_message = str(e)
            self.handle_sync_failure("HTTPStatusError", e, error_message)
        except Exception as e:
            self.handle_sync_failure("Error", e)

    def handle_sync_failure(self, failure_type, exception, message=None):
        self.status = IPFabricSyncStatusChoices.FAILED

        if message:
            self.logger.log_failure(
                f"{message} ({failure_type}): `{exception}`", obj=self
            )
        else:
            self.logger.log_failure(f"Syncing Snapshot Failed: `{exception}`", obj=self)


class IPFabricSource(IPFabricClient, JobsMixin, PrimaryModel):
    name = models.CharField(max_length=100, unique=True)
    type = models.CharField(
        verbose_name=_("type"),
        max_length=50,
        choices=IPFabricSourceTypeChoices,
        default=IPFabricSourceTypeChoices.LOCAL,
    )
    url = models.CharField(max_length=200, verbose_name=_("URL"))
    status = models.CharField(
        max_length=50,
        choices=IPFabricSourceStatusChoices,
        default=IPFabricSourceStatusChoices.NEW,
        editable=False,
    )
    parameters = models.JSONField(blank=True, null=True)
    last_synced = models.DateTimeField(blank=True, null=True, editable=True)

    class Meta:
        ordering = ("name",)
        verbose_name = _("IP Fabric Source")
        verbose_name_plural = _("IP Fabric Sources")

    def __str__(self):
        return f"{self.name}"

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricsource", args=[self.pk])

    @property
    def ready_for_sync(self):
        return self.status not in (
            IPFabricSourceStatusChoices.QUEUED,
            IPFabricSourceStatusChoices.SYNCING,
        )

    @property
    def docs_url(self):
        # TODO: Add docs url
        return ""

    def clean(self):
        super().clean()

        self.url = self.url.rstrip("/")

    def enqueue_sync_job(self, request):
        # Set the status to "syncing"
        self.status = IPFabricSourceStatusChoices.QUEUED
        IPFabricSource.objects.filter(pk=self.pk).update(status=self.status)

        # Enqueue a sync job
        return Job.enqueue(
            import_string("ipfabric_netbox.jobs.sync_ipfabricsource"),
            name=f"{self.name} Snapshot Sync",
            instance=self,
            user=request.user,
        )

    def sync(self, job):
        self.logger = SyncLogging(job=job.pk)
        if self.status == IPFabricSourceStatusChoices.SYNCING:
            self.logger.log_failure(
                "Cannot initiate sync; syncing already in progress.", obj=self
            )
            raise SyncError("Cannot initiate sync; syncing already in progress.")

        pre_sync.send(sender=self.__class__, instance=self)

        self.status = IPFabricSourceStatusChoices.SYNCING
        IPFabricSource.objects.filter(pk=self.pk).update(status=self.status)

        # Begin Sync
        try:
            self.logger.log_info(f"Syncing snapshots from {self.name}", obj=self)
            logger.debug(f"Syncing snapshots from {self.url}")

            self.parameters["base_url"] = self.url
            ipf = self.get_client(parameters=self.parameters)

            if not ipf:
                raise SyncError("Unable to connect to IP Fabric.")

            for snapshot_id, value in ipf.snapshots.items():
                if snapshot_id not in ["$prev", "$lastLocked"]:
                    if value.name:
                        name = (
                            value.name
                            + " - "
                            + value.start.strftime("%d-%b-%y %H:%M:%S")
                        )
                    else:
                        name = value.start.strftime("%d-%b-%y %H:%M:%S")

                    if value.status == "done":
                        status = "loaded"
                    else:
                        status = value.status

                    data = {
                        "name": name,
                        "data": json.loads(value.model_dump_json(exclude={"client"})),
                        "date": value.start,
                        "created": timezone.now(),
                        "last_updated": timezone.now(),
                        "status": status,
                    }
                    snapshot, _ = IPFabricSnapshot.objects.update_or_create(
                        source=self, snapshot_id=snapshot_id, defaults=data
                    )
                    self.logger.log_info(
                        f"Created/Updated Snapshot {snapshot.name} ({snapshot.snapshot_id})",
                        obj=snapshot,  # noqa E225
                    )
            self.status = IPFabricSourceStatusChoices.COMPLETED
            self.logger.log_success(f"Completed syncing snapshots from {self.name}")
            logger.debug(f"Completed syncing snapshots from {self.url}")
        except Exception as e:
            self.handle_sync_failure(type(e).__name__, e)
        finally:
            self.last_synced = timezone.now()
            IPFabricSource.objects.filter(pk=self.pk).update(
                status=self.status, last_synced=self.last_synced
            )
            self.logger.log_info("Sync job completed.", obj=self)
            if job:
                job.data = self.logger.log_data
        # Emit the post_sync signal
        # post_sync.send(sender=self.__class__, instance=self)

    @classmethod
    def get_for_site(cls, site: Site):
        """Get all snapshots containing the given site."""
        return cls.objects.filter(
            Q(snapshots__data__sites__contains=[site.name])
        ).distinct()


class IPFabricSnapshot(TagsMixin, ChangeLoggedModel):
    source = models.ForeignKey(
        to=IPFabricSource,
        on_delete=models.CASCADE,
        related_name="snapshots",
        editable=False,
    )
    name = models.CharField(max_length=200)
    snapshot_id = models.CharField(max_length=100)
    data = models.JSONField(blank=True, null=True)
    date = models.DateTimeField(blank=True, null=True, editable=False)
    status = models.CharField(
        max_length=50,
        choices=IPFabricSnapshotStatusModelChoices,
        default=IPFabricSnapshotStatusModelChoices.STATUS_UNLOADED,
    )

    objects = RestrictedQuerySet.as_manager()

    class Meta:
        ordering = ("source", "-date")
        verbose_name = _("IP Fabric Snapshot")
        verbose_name_plural = _("IP Fabric Snapshots")

    def __str__(self):
        return f"{self.name} - {self.snapshot_id}"

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricsnapshot", args=[self.pk])

    def get_status_color(self):
        return IPFabricSnapshotStatusModelChoices.colors.get(self.status)

    @property
    def sites(self):
        if self.data:
            sites = self.data.get("sites", None)
            if sites:
                return sites
            else:
                return []
        else:
            return []


class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
    objects = RestrictedQuerySet.as_manager()
    name = models.CharField(max_length=100, unique=True)
    snapshot_data = models.ForeignKey(
        to=IPFabricSnapshot,
        on_delete=models.CASCADE,
        related_name="snapshots",
    )
    status = models.CharField(
        max_length=50,
        choices=IPFabricSyncStatusChoices,
        default=IPFabricSyncStatusChoices.NEW,
        editable=False,
    )
    parameters = models.JSONField(blank=True, null=True)
    auto_merge = models.BooleanField(default=False)
    update_custom_fields = models.BooleanField(default=True)
    last_synced = models.DateTimeField(blank=True, null=True, editable=False)
    scheduled = models.DateTimeField(null=True, blank=True)
    interval = models.PositiveIntegerField(
        blank=True,
        null=True,
        validators=(MinValueValidator(1),),
        help_text=_("Recurrence interval (in minutes)"),
    )
    user = models.ForeignKey(
        to=settings.AUTH_USER_MODEL,
        on_delete=models.SET_NULL,
        related_name="+",
        blank=True,
        null=True,
    )

    class Meta:
        ordering = ["pk"]
        verbose_name = _("IP Fabric Sync")

    def __str__(self):
        return f"{self.name}"

    @property
    def docs_url(self):
        # TODO: Add docs url
        return ""

    @property
    def logger(self):
        return getattr(self, "_logger", SyncLogging(job=self.pk))

    @logger.setter
    def logger(self, value):
        self._logger = value

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricsync", args=[self.pk])

    def get_status_color(self):
        return IPFabricSyncStatusChoices.colors.get(self.status)

    def clean(self):
        super().clean()

        # Skip validation if parameters is None or empty
        if not self.parameters:
            return

        # Valid special keys
        valid_special_keys = {"sites", "groups"}
        errors = {}

        for key, value in self.parameters.items():
            # Check if it's a special key
            if key in valid_special_keys:
                # Validate value types for special keys
                if key == "sites":
                    if not isinstance(value, list):
                        errors[
                            key
                        ] = f"Parameter 'sites' must be a list, got {type(value).__name__}"
                elif key == "groups":
                    if not isinstance(value, list):
                        errors[
                            key
                        ] = f"Parameter 'groups' must be a list, got {type(value).__name__}"
                    elif value and not all(isinstance(item, int) for item in value):
                        errors[key] = "Parameter 'groups' must be a list of integers"
            # Check if it's a ContentType reference (app_label.model_name)
            elif "." in key:
                parts = key.split(".")
                if len(parts) != 2:
                    errors[key] = (
                        f"Invalid parameter key '{key}'. ContentType format must be "
                        "'app_label.model_name' (e.g., 'dcim.device', 'ipam.vlan')"
                    )
                else:
                    app_label, model_name = parts
                    # Validate ContentType exists
                    try:
                        ContentType.objects.get(app_label=app_label, model=model_name)
                    except ContentType.DoesNotExist:
                        errors[key] = (
                            f"ContentType '{key}' does not exist. Expected format: "
                            "'app_label.model_name' (e.g., 'dcim.device', 'ipam.vlan')"
                        )

                    # Validate value type for ContentType keys (must be boolean)
                    if not isinstance(value, bool):
                        errors[
                            key
                        ] = f"Parameter '{key}' must be a boolean, got {type(value).__name__}"
            else:
                # Invalid key format
                errors[key] = (
                    f"Invalid parameter key '{key}'. Must be 'sites', 'groups', or "
                    "valid ContentType format 'app_label.model_name' (e.g., 'dcim.device', 'ipam.vlan')"
                )

        if errors:
            raise ValidationError(errors)

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        if self.scheduled:
            self.enqueue_sync_job()

    @property
    def ready_for_sync(self):
        if self.status not in (IPFabricSyncStatusChoices.SYNCING,):
            if self.snapshot_data.source.type == "remote":
                if self.snapshot_data.ipf_data.count() > 0:
                    return True
                else:
                    return False
            else:
                return True
        else:
            return False

    @property
    def last_ingestion(self):
        return self.ipfabricingestion_set.last()

    @staticmethod
    def get_transform_maps(
        group_ids: list[int] | None = None,
    ) -> QuerySet["IPFabricTransformMap"]:
        """
        Returns a queryset of IPFabricTransformMap objects that would be used by this sync,
        following group and default precedence logic.

        Transform maps are unique by (target_model, source_endpoint) combination.
        Groups have precedence: later groups override earlier groups and defaults.
        """
        default_maps = IPFabricTransformMap.objects.filter(group__isnull=True)
        group_ids = group_ids or []

        # Use composite key: (target_model_id, source_endpoint_id)
        maps_by_composite_key = {
            (tm.target_model_id, tm.source_endpoint_id): tm for tm in default_maps
        }

        # Replace default maps with the ones from the groups, in given order.
        if group_ids:
            for group_id in group_ids:
                group_maps = IPFabricTransformMap.objects.filter(group_id=group_id)
                for tm in group_maps:
                    maps_by_composite_key[
                        (tm.target_model_id, tm.source_endpoint_id)
                    ] = tm

        return IPFabricTransformMap.objects.filter(
            pk__in=[tm.pk for tm in maps_by_composite_key.values()]
        )

    @classmethod
    def get_model_hierarchy(
        cls, group_ids: list[int] | None = None
    ) -> list["IPFabricTransformMap"]:
        """
        Get transform maps in hierarchical order based on parent relationships.
        Uses topological sort (Kahn's algorithm) to support multiple parents.
        Transform maps without parents come first, then their children, etc.

        Example: IP Address transform map has parents [Interface, VRF], so it will only be
        processed after both Interface AND VRF transform maps have been processed.

        Returns list of transform maps ordered by dependencies.
        """
        maps = cls.get_transform_maps(group_ids)

        # Build adjacency list and in-degree count using transform map IDs
        graph = {}  # parent_tm_id -> [child_tm_id, ...]
        in_degree = {}  # tm_id -> count of unprocessed parents
        tm_by_id = {}  # tm_id -> transform_map

        for transform_map in maps:
            tm_id = transform_map.id
            tm_by_id[tm_id] = transform_map

            # Get all parents for this transform map
            parent_maps = transform_map.parents.all()

            # Set in-degree (number of parents)
            in_degree[tm_id] = parent_maps.count()

            # Build adjacency list (parent -> children)
            for parent_map in parent_maps:
                parent_id = parent_map.id
                graph.setdefault(parent_id, []).append(tm_id)

        # Topological sort using Kahn's algorithm (BFS-based)
        queue = [tm_id for tm_id, degree in in_degree.items() if degree == 0]
        ordered = []

        while queue:
            # Pop from front to maintain BFS/level-order
            current_tm_id = queue.pop(0)
            ordered.append(current_tm_id)

            # Reduce in-degree for all children
            for child_tm_id in graph.get(current_tm_id, []):
                in_degree[child_tm_id] -= 1
                if in_degree[child_tm_id] == 0:
                    queue.append(child_tm_id)

        # Check for circular dependencies
        if len(ordered) != len(in_degree):
            unprocessed_ids = set(in_degree.keys()) - set(ordered)
            unprocessed_maps = [tm_by_id[tm_id] for tm_id in unprocessed_ids]
            raise ValidationError(
                f"Circular dependency detected in transform map hierarchy. "
                f"Unprocessed maps: {', '.join(str(tm) for tm in unprocessed_maps)}"
            )

        # Return ordered list of transform maps
        return [tm_by_id[tm_id] for tm_id in ordered]

    def get_enabled_endpoints(self) -> QuerySet["IPFabricEndpoint"]:
        """
        Get the IPFabricEndpoint objects that are used by this sync.

        Returns endpoints that:
        1. Are source endpoints in transform maps for this sync's groups
        2. Have transform maps targeting models enabled in sync parameters
        3. Automatically includes dependency endpoints:
           - DEVICES: Included when VIRTUALCHASSIS, INTERFACE, MACADDRESS, or IPADDRESS is enabled
           - INTERFACES: Included when IPADDRESSES is enabled (IP addresses need interfaces)
           - VIRTUALCHASSIS + VSS_CHASSIS: Included when DEVICE is enabled (devices may be in VC/VSS)
           - VSS_CHASSIS: Also included when VIRTUALCHASSIS is enabled (different chassis stacking types)

        Returns:
            QuerySet of IPFabricEndpoint objects
        """
        # Get enabled models from sync parameters
        enabled_models = [
            param
            for param in self.parameters.keys()
            if "." in param and self.parameters.get(param) is True
        ]
        if not enabled_models:
            return IPFabricEndpoint.objects.none()

        # Get transform maps for this sync
        group_ids = self.parameters.get("groups", [])
        transform_maps = self.get_transform_maps(group_ids=group_ids)

        # Get unique endpoints that have transform maps with enabled target models
        endpoint_ids = set()
        for tm in transform_maps:
            model_string = f"{tm.target_model.app_label}.{tm.target_model.model}"
            if model_string in enabled_models:
                endpoint_ids.add(tm.source_endpoint_id)

        # Add required dependency endpoints
        # TODO: Remove this when we can specify multiple endpoints in transform maps and properly define dependencies there

        # Include DEVICES endpoint only for specific models that require it but don't have it in transform maps
        if (
            IPFabricSyncParameterChoices.VIRTUALCHASSIS in enabled_models
            or IPFabricSyncParameterChoices.INTERFACE in enabled_models
            or IPFabricSyncParameterChoices.MACADDRESS in enabled_models
            or IPFabricSyncParameterChoices.IPADDRESS in enabled_models
        ):
            device_endpoint = IPFabricEndpoint.objects.filter(
                endpoint=IPFabricEndpointChoices.DEVICES
            ).first()
            if device_endpoint:
                endpoint_ids.add(device_endpoint.pk)

        # When IPADDRESSES is enabled, also include INTERFACES endpoint
        # (IP addresses require interface assignments)
        if IPFabricSyncParameterChoices.IPADDRESS in enabled_models:
            interface_endpoint = IPFabricEndpoint.objects.filter(
                endpoint=IPFabricEndpointChoices.INTERFACES
            ).first()
            if interface_endpoint:
                endpoint_ids.add(interface_endpoint.pk)

        # When DEVICE is enabled, include both VIRTUALCHASSIS and VSS_CHASSIS endpoints
        # (devices may be part of virtual chassis or VSS configurations)
        if IPFabricSyncParameterChoices.DEVICE in enabled_models:
            vc_endpoint = IPFabricEndpoint.objects.filter(
                endpoint=IPFabricEndpointChoices.VIRTUALCHASSIS
            ).first()
            if vc_endpoint:
                endpoint_ids.add(vc_endpoint.pk)

            vss_endpoint = IPFabricEndpoint.objects.filter(
                endpoint=IPFabricEndpointChoices.VSS_CHASSIS
            ).first()
            if vss_endpoint:
                endpoint_ids.add(vss_endpoint.pk)

        # When VIRTUALCHASSIS is enabled, also include VSS_CHASSIS endpoint
        # (both endpoints represent different types of chassis stacking)
        if IPFabricSyncParameterChoices.VIRTUALCHASSIS in enabled_models:
            vss_endpoint = IPFabricEndpoint.objects.filter(
                endpoint=IPFabricEndpointChoices.VSS_CHASSIS
            ).first()
            if vss_endpoint:
                endpoint_ids.add(vss_endpoint.pk)

        return IPFabricEndpoint.objects.filter(pk__in=endpoint_ids)

    def delete_scheduled_jobs(self) -> None:
        Job.objects.filter(
            object_type=ObjectType.objects.get_for_model(self),
            object_id=self.pk,
            status__in=[
                JobStatusChoices.STATUS_PENDING,
                JobStatusChoices.STATUS_SCHEDULED,
            ],
        ).delete()

    def enqueue_sync_job(self, adhoc=False, user=None) -> Job | None:
        def set_syncing_status():
            self.status = IPFabricSyncStatusChoices.QUEUED
            IPFabricSync.objects.filter(pk=self.pk).update(status=self.status)

        def sync_snapshots():
            Job.enqueue(
                import_string("ipfabric_netbox.jobs.sync_ipfabricsource"),
                name=f"{self.name} Snapshot Sync (Pre Ingestion)",
                instance=self.snapshot_data.source,
                user=self.user,
            )

        # Enqueue a sync job
        if not user:
            user = self.user

        if not adhoc:
            if self.scheduled:
                # We want to schedule a recurring Job
                # We need to replace the old scheduled Job to make sure it has current context
                self.delete_scheduled_jobs()
                set_syncing_status()
                sync_snapshots()
                job = Job.enqueue(
                    import_string("ipfabric_netbox.jobs.sync_ipfabric"),
                    name=f"{self.name} - (scheduled)",
                    instance=self,
                    user=self.user,
                    schedule_at=self.scheduled,
                    interval=self.interval,
                    ipf_version=NetboxIPFabricConfig.version,
                )
            else:
                # There should be no scheduled Job anymore, clean it up
                self.delete_scheduled_jobs()
                job = None
        else:
            # Start adhoc job immediately
            set_syncing_status()
            sync_snapshots()
            job = Job.enqueue(
                import_string("ipfabric_netbox.jobs.sync_ipfabric"),
                instance=self,
                user=user,
                name=f"{self.name} - (adhoc)",
                adhoc=adhoc,
                ipf_version=NetboxIPFabricConfig.version,
            )
        return job

    def sync(self, job=None, main_version: str | None = None):
        if job:
            self.logger = SyncLogging(job=job.pk)
            user = job.user
        else:
            self.logger = SyncLogging(job=self.pk)
            user = self.user

        maps = self.get_transform_maps(self.parameters.get("groups", []))
        missing = []
        for app_label, model in required_transform_map_contenttypes:
            if not maps.filter(
                target_model=ContentType.objects.get(app_label=app_label, model=model)
            ):
                missing.append(f"{app_label}.{model}")
        if missing:
            self.logger.log_failure(
                f"Combination of these transform map groups failed validation. Missing maps: {missing}.",
                obj=self,
            )
            raise SyncError(
                f"Combination of these transform map groups failed validation. Missing maps: {missing}."
            )

        if self.status == IPFabricSyncStatusChoices.SYNCING:
            raise SyncError("Cannot initiate sync; ingestion already in progress.")

        worker_version = NetboxIPFabricConfig.version
        if main_version is not None and not main_version == worker_version:
            log_msg = f"The IPFabric plugin version differs between main NetBox and this worker. Make sure to update both when updating. Main version: {main_version}, Worker version: {worker_version}."
            self.logger.log_warning(log_msg)
            logger.warning(log_msg)

        pre_sync.send(sender=self.__class__, instance=self)

        self.status = IPFabricSyncStatusChoices.SYNCING
        IPFabricSync.objects.filter(pk=self.pk).update(status=self.status)

        # Begin Sync
        self.logger.log_info(
            f"Ingesting data from {self.snapshot_data.source.name}", obj=self
        )
        logger.info(f"Ingesting data from {self.snapshot_data.source.name}")

        self.snapshot_data.source.parameters["base_url"] = self.snapshot_data.source.url
        self.parameters["snapshot_id"] = self.snapshot_data.snapshot_id
        self.logger.log_info(
            f"Syncing with the following data {json.dumps(self.parameters)}", obj=self
        )
        logger.info(f"Syncing with the following data {json.dumps(self.parameters)}")

        current_time = str(timezone.now())
        ingestion = IPFabricIngestion.objects.create(sync=self, job=job)
        try:
            branch = Branch(name=f"IP Fabric Sync {self.name} - {current_time}")
            branch.save(provision=False)
            ingestion.branch = branch
            ingestion.save()

            if job:
                # Re-assign the Job from IPFSync to IPFabricIngestion so it is listed in the ingestion
                job.object_type = ObjectType.objects.get_for_model(ingestion)
                job.object_id = ingestion.pk
                job.save()
            branch.provision(user=user)
            branch.refresh_from_db()
            if branch.status == BranchStatusChoices.FAILED:
                print("Branch Failed")
                self.logger.log_failure(f"Branch Failed: `{branch}`", obj=branch)
                raise SyncError("Branch Creation Failed")

            self.logger.log_info(f"New branch Created {branch.name}", obj=branch)
            logger.info(f"New branch Created {branch.name}")

            self.logger.log_info("Fetching IP Fabric Client", obj=branch)
            logger.info("Fetching IP Fabric Client")

            if self.snapshot_data.source.type == IPFabricSourceTypeChoices.LOCAL:
                ipf = self.get_client(parameters=self.snapshot_data.source.parameters)
                if not ipf:
                    logger.debug("Unable to connect to IP Fabric.")
                    raise SyncError("Unable to connect to IP Fabric.")
            else:
                ipf = None

            runner = IPFabricSyncRunner(
                client=ipf,
                ingestion=ingestion,
                settings=self.parameters,
                sync=self,
            )

            # Not using `deactivate_branch` since that does not clean up on Exception
            current_branch = active_branch.get()
            if not (token := current_request.get()):
                # This allows for ChangeLoggingMiddleware to create ObjectChanges
                token = current_request.set(
                    NetBoxFakeRequest({"id": uuid4(), "user": user})
                )
            try:
                active_branch.set(branch)
                try:
                    try:
                        signals.post_save.connect(
                            assign_primary_mac_address, sender=MACAddress
                        )
                        runner.collect_and_sync(
                            ingestion=IPFabricIngestion.objects.get(pk=ingestion.pk)
                        )
                    finally:
                        signals.post_save.disconnect(
                            assign_primary_mac_address, sender=MACAddress
                        )
                finally:
                    active_branch.set(None)
            finally:
                current_request.set(token.old_value)
                active_branch.set(current_branch)

            if self.status != IPFabricSyncStatusChoices.FAILED:
                self.status = IPFabricSyncStatusChoices.READY_TO_MERGE
        except JobTimeoutException as e:
            self.status = IPFabricSyncStatusChoices.TIMEOUT
            self.logger.log_failure(
                f"Sync Job Timed Out: `{e}`. Check if RQ_DEFAULT_TIMEOUT needs increasing.",
                obj=ingestion,
            )
            logger.debug(
                f"Sync Job Timed Out: `{e}`. Check if RQ_DEFAULT_TIMEOUT needs increasing."
            )
        except Exception as e:
            self.status = IPFabricSyncStatusChoices.FAILED
            self.logger.log_failure(f"Ingestion Failed: `{e}`", obj=ingestion)
            self.logger.log_failure(
                f"Stack Trace: `{traceback.format_exc()}`", obj=ingestion
            )
            logger.debug(f"Ingestion Failed: `{e}`")

        logger.debug(f"Completed ingesting data from {self.snapshot_data.source.name}")
        self.logger.log_info(
            f"Completed ingesting data from {self.snapshot_data.source.name}", obj=self
        )

        self.last_synced = timezone.now()

        if self.auto_merge and self.status == IPFabricSyncStatusChoices.READY_TO_MERGE:
            self.logger.log_info("Auto Merging Ingestion", obj=ingestion)
            logger.info("Auto Merging Ingestion")
            try:
                ingestion.enqueue_merge_job(user=user, remove_branch=True)
                self.logger.log_info("Auto Merge Job Enqueued", obj=ingestion)
                logger.info("Auto Merge Job Enqueued")
            except NameError:
                self.logger.log_failure(
                    "Failed to Auto Merge, IPFabricIngestion does not exist",
                    obj=ingestion,
                )
                logger.debug("Failed to Auto Merge, IPFabricIngestion does not exist")

        IPFabricSync.objects.filter(pk=self.pk).update(
            status=self.status, last_synced=self.last_synced
        )
        if job:
            job.data = self.logger.log_data


class IPFabricIngestion(JobsMixin, models.Model):
    """
    Links IP Fabric Sync to its Branches.
    """

    objects = RestrictedQuerySet.as_manager()

    sync = models.ForeignKey(IPFabricSync, on_delete=models.CASCADE)
    job = models.ForeignKey(Job, on_delete=models.SET_NULL, null=True)
    branch = models.OneToOneField(Branch, on_delete=models.SET_NULL, null=True)

    class Meta:
        ordering = ("pk",)
        verbose_name = _("IP Fabric Ingestion")
        verbose_name_plural = _("IP Fabric Ingestions")

    def __str__(self):
        return self.name

    @property
    def name(self):
        if self.branch:
            return self.branch.name
        try:
            return f"{self.sync.name} (Ingestion {self.pk})"
        except IPFabricIngestion.sync.RelatedObjectDoesNotExist:
            return f"Ingestion {self.pk} (No Sync)"

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricingestion", args=[self.pk])

    def enqueue_merge_job(self, user, remove_branch=False):
        # Set the status to "queued"
        self.status = IPFabricSyncStatusChoices.QUEUED
        IPFabricSync.objects.filter(ipfabricingestion=self.pk).update(
            status=self.status
        )

        # Enqueue a sync job
        return Job.enqueue(
            import_string("ipfabric_netbox.jobs.merge_ipfabric_ingestion"),
            name=f"{self.name} Merge",
            instance=self,
            user=user,
            remove_branch=remove_branch,
        )

    def get_logs(self):
        if not self.job:
            # The Job is deleted by manual action
            return {}
        if self.job.data:
            job_results = self.job.data
        else:
            job_results = cache.get(f"ipfabric_sync_{self.job.pk}")
            if not job_results:
                job_results = cache.get(f"ipfabric_sync_{self.sync.pk}")
        return job_results

    def get_statistics(self):
        job_results = self.get_logs()
        statistics = {}
        if job_results:
            for model, stats in job_results["statistics"].items():
                if not stats["total"]:
                    continue
                if stats["total"] > 0:
                    statistics[model] = stats["current"] / stats["total"] * 100
                else:
                    statistics[model] = stats["current"] / 1 * 100

        # Sort statistics according to transform map hierarchy
        # This ensures consistent ordering in the progress display matching sync order
        try:
            group_ids = self.sync.parameters.get("groups", [])
            transform_maps = self.sync.get_model_hierarchy(group_ids=group_ids)

            # Build list of model strings in hierarchical order
            hierarchy_models = []
            for transform_map in transform_maps:
                model_string = f"{transform_map.target_model.app_label}.{transform_map.target_model.model}"
                if model_string in statistics:
                    hierarchy_models.append(model_string)

            # Group models by app_label while preserving hierarchical order within each group

            app_groups = defaultdict(list)
            for model_string in hierarchy_models:
                app_label = model_string.split(".")[0]
                app_groups[app_label].append(model_string)

            # Sort app_labels alphabetically and create ordered statistics
            ordered_statistics = {}
            for app_label in sorted(app_groups.keys()):
                for model_string in app_groups[app_label]:
                    ordered_statistics[model_string] = statistics[model_string]

            # Add any remaining statistics that weren't in the hierarchy (sorted)
            remaining_models = sorted(
                [m for m in statistics.keys() if m not in ordered_statistics]
            )
            for model_string in remaining_models:
                ordered_statistics[model_string] = statistics[model_string]

            statistics = ordered_statistics
        except Exception:
            # If hierarchy ordering fails, fall back to alphabetical sorting
            statistics = dict(sorted(statistics.items()))

        return {"job_results": job_results, "statistics": statistics}

    def sync_merge(self):
        ipfabricsync = self.sync
        if ipfabricsync.status == IPFabricSyncStatusChoices.MERGING:
            raise SyncError("Cannot initiate merge; merge already in progress.")

        pre_sync.send(sender=self.__class__, instance=self)

        ipfabricsync.status = IPFabricSyncStatusChoices.MERGING
        IPFabricSync.objects.filter(ipfabricingestion=self.pk).update(
            status=self.sync.status
        )

        # Begin Sync
        logger.debug(f"Merging {self.name}")
        try:
            self.branch.merge(user=self.sync.user)
            ipfabricsync.status = IPFabricSyncStatusChoices.COMPLETED
        except Exception as e:
            ipfabricsync.status = IPFabricSyncStatusChoices.FAILED
            logger.debug(f"Merging {self.name} Failed: `{e}`")

        logger.debug(f"Completed merge {self.name}")

        ipfabricsync.last_synced = timezone.now()
        IPFabricSync.objects.filter(ipfabricingestion=self.pk).update(
            status=ipfabricsync.status, last_synced=ipfabricsync.last_synced
        )


class IPFabricIngestionIssue(models.Model):
    objects = RestrictedQuerySet.as_manager()

    ingestion = models.ForeignKey(
        to="IPFabricIngestion", on_delete=models.CASCADE, related_name="issues"
    )
    timestamp = models.DateTimeField(default=timezone.now)
    model = models.CharField(max_length=100, blank=True, null=True)
    message = models.TextField()
    raw_data = models.TextField(blank=True, default="")
    coalesce_fields = models.TextField(blank=True, default="")
    defaults = models.TextField(blank=True, default="")
    exception = models.TextField()

    class Meta:
        ordering = ["timestamp"]
        verbose_name = _("IP Fabric Ingestion Issue")
        verbose_name_plural = _("IP Fabric Ingestion Issues")

    def __str__(self):
        return f"[{self.timestamp}] {self.message}"


class IPFabricData(models.Model):
    snapshot_data = models.ForeignKey(
        to=IPFabricSnapshot,
        on_delete=models.CASCADE,
        related_name="ipf_data",
    )
    data = models.JSONField(blank=True, null=True)
    type = models.CharField(
        max_length=50,
        choices=IPFabricRawDataTypeChoices,
    )
    objects = RestrictedQuerySet.as_manager()

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricdata_data", args=[self.pk])


class IPFabricFilter(NetBoxModel):
    objects = RestrictedQuerySet.as_manager()

    name = models.CharField(max_length=100, unique=True)
    description = models.TextField(blank=True, null=True)
    endpoints = models.ManyToManyField(
        to=IPFabricEndpoint,
        related_name="filters",
        editable=True,
        default=None,
        blank=True,
    )
    filter_type = models.CharField(
        max_length=10, choices=IPFabricFilterTypeChoices, verbose_name=_("Filter Type")
    )
    syncs = models.ManyToManyField(
        to=IPFabricSync,
        related_name="filters",
        editable=True,
        default=None,
        blank=True,
    )

    class Meta:
        ordering = ("pk",)
        verbose_name = _("IP Fabric Filter")
        verbose_name_plural = _("IP Fabric Filters")

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse("plugins:ipfabric_netbox:ipfabricfilter", args=[self.pk])

    def merge_expressions(self) -> list[dict]:
        """Merge all linked Expressions into a single filter expression."""
        merged_expression = []
        for expression in self.expressions.all():
            merged_expression.extend(expression.expression)
        return merged_expression


class IPFabricFilterExpression(NetBoxModel):
    objects = RestrictedQuerySet.as_manager()

    name = models.CharField(max_length=100, unique=True)
    description = models.TextField(blank=True, null=True)
    expression = models.JSONField(
        blank=False,
        null=False,
        default=list,
        verbose_name=_("IP Fabric Filter Expression JSON"),
        help_text=_(
            "JSON filter for API call to IPF, can be obtained from IPF UI call via browser developer console."
        ),
    )
    filters = models.ManyToManyField(
        to=IPFabricFilter,
        related_name="expressions",
        editable=True,
    )

    class Meta:
        ordering = ("pk",)
        verbose_name = _("IP Fabric Filter Expression")
        verbose_name_plural = _("IP Fabric Filter Expressions")

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse(
            "plugins:ipfabric_netbox:ipfabricfilterexpression", args=[self.pk]
        )

    def clean(self):
        super().clean()

        # Validate that expression is a list of dictionaries
        if self.expression is None:
            raise ValidationError({"expression": _("Filter Expression is required.")})

        if not isinstance(self.expression, list):
            raise ValidationError(
                {
                    "expression": _("Expression must be a list. Got: %(type)s")
                    % {"type": type(self.expression).__name__}
                }
            )

        if not self.expression:
            raise ValidationError(
                {"expression": _("Expression cannot be an empty list.")}
            )

        for idx, item in enumerate(self.expression):
            if not isinstance(item, dict):
                raise ValidationError(
                    {
                        "expression": _(
                            "Expression item at index %(index)d must be a dictionary. Got: %(type)s"
                        )
                        % {"index": idx, "type": type(item).__name__}
                    }
                )


@receiver(m2m_changed, sender=IPFabricTransformMap.parents.through)
def validate_circular_dependency_on_m2m_change(
    sender, instance, action, pk_set, **kwargs
):
    """
    Validate circular dependencies when parent M2M relationships are modified.
    This catches changes made through the API or programmatically.
    """
    if action == "pre_add" and pk_set:
        # Simulate what the parents would be after this add operation
        current_parent_ids = set(instance.parents.values_list("pk", flat=True))
        future_parent_ids = current_parent_ids | pk_set

        # Get the actual parent objects
        future_parents = IPFabricTransformMap.objects.filter(pk__in=future_parent_ids)

        # Run cycle detection with the future parent set
        def get_parents(
            node_id: int, parent_override: models.QuerySet | None
        ) -> models.QuerySet:
            """Get parents for a node, with optional override for the instance being modified."""
            if node_id == instance.pk and parent_override is not None:
                # Use the future parents for the current node
                return parent_override
            else:
                # Use existing parents for other nodes
                node = IPFabricTransformMap.objects.get(pk=node_id)
                return node.parents.all()

        if has_cycle_dfs(instance.pk, get_parents, parent_override=future_parents):
            raise ValidationError(
                _(
                    "Cannot add these parents: circular dependency detected. "
                    "A transform map cannot be an ancestor of itself."
                )
            )
