import copy
import json
from unittest.mock import MagicMock
from unittest.mock import patch

from dcim.models import Device
from django.contrib.auth import get_user_model
from django.contrib.contenttypes.models import ContentType
from django.core import serializers
from django.core.exceptions import ValidationError
from django.db import DEFAULT_DB_ALIAS
from django.test import TestCase
from django.utils import timezone

from ipfabric_netbox.choices import IPFabricFilterTypeChoices
from ipfabric_netbox.choices import IPFabricIngestionPhaseChoices
from ipfabric_netbox.choices import IPFabricSnapshotStatusModelChoices
from ipfabric_netbox.models import IPFabricEndpoint
from ipfabric_netbox.models import IPFabricFilter
from ipfabric_netbox.models import IPFabricFilterExpression
from ipfabric_netbox.models import IPFabricIngestion
from ipfabric_netbox.models import IPFabricSnapshot
from ipfabric_netbox.models import IPFabricSource
from ipfabric_netbox.models import IPFabricSync
from ipfabric_netbox.models import IPFabricTransformField
from ipfabric_netbox.models import IPFabricTransformMap
from ipfabric_netbox.models import IPFabricTransformMapGroup
from ipfabric_netbox.utilities.ipfutils import IPFabricSyncRunner
from ipfabric_netbox.utilities.logging import SyncLogging
from ipfabric_netbox.utilities.merge import merge_branch


class IPFabricTransformMapModelTestCase(TestCase):
    @classmethod
    def setUpTestData(cls):
        # Endpoints and content types used by the get_model_hierarchy tests below.
        cls.ep_parts = IPFabricEndpoint.objects.get(endpoint="/inventory/part-numbers")
        cls.ep_ifaces = IPFabricEndpoint.objects.get(endpoint="/inventory/interfaces")
        cls.ep_managed_ip = IPFabricEndpoint.objects.get(
            endpoint="/technology/addressing/managed-ip/ipv4"
        )
        cls.ct_inventory_item = ContentType.objects.get(
            app_label="dcim", model="inventoryitem"
        )
        cls.ct_interface = ContentType.objects.get(app_label="dcim", model="interface")
        cls.ct_mac = ContentType.objects.get(app_label="dcim", model="macaddress")
        cls.ct_ipaddress = ContentType.objects.get(app_label="ipam", model="ipaddress")

    # ------------------------------------------------------------------
    # Helpers used by the get_model_hierarchy tests
    # ------------------------------------------------------------------

    def _default_map(self, name, endpoint, ct):
        """Create a default (no-group) transform map."""
        return IPFabricTransformMap.objects.create(
            name=name,
            source_endpoint=endpoint,
            target_model=ct,
            group=None,
        )

    def _group_map(self, name, endpoint, ct, group):
        """Create a group-owned transform map."""
        return IPFabricTransformMap.objects.create(
            name=name,
            source_endpoint=endpoint,
            target_model=ct,
            group=group,
        )

    def setUp(self):
        source = IPFabricSource.objects.create(
            name="test",
            url="https://localhost",
            status="new",
            parameters={"auth": "token123", "verify": True},
        )
        snapshot = IPFabricSnapshot.objects.create(
            name="S01 - Day 2 - IPF Lab - 02-Jul-21 06:29:16 - 12dd8c61-129c-431a-b98b-4c9211571f89",
            source=source,
            snapshot_id="12dd8c61-129c-431a-b98b-4c9211571f89",
            status=IPFabricSnapshotStatusModelChoices.STATUS_LOADED,
            data={
                "end": "2021-07-02T06:29:16.311000Z",
                "name": "S01 - Day 2 - IPF Lab",
                "note": "Multi-Environment containing:\\nAWS, Azure, ACI, NSX-T, Viptela, Versa, SilverPeak, Meraki\\n\\nSite 48 - devices added, NTP issue\\nSite 68 &38 - NTP update\\nSite 38 - resiliency affected (no ospfx2 - no L1 link x1) + passive interfaces FIXED / NTP partial update\\n?E2E: 38 - 66 - migration HTTP to HTTPS\\n?Site 66 - FW bypass E2E\\nVRRP improvements (LAB1 / L52)",
                "sites": [
                    "35COLO",
                    "35HEADOFFICE",
                    "35PRODUCTION",
                    "35SALES",
                    "38 Pilsen DR",
                    "66 Ostrava DC",
                    "68 Pardubice Distribution",
                    "ACI",
                    "AWS_SITE",
                    "AZURE",
                    "HWLAB",
                    "L31",
                    "L33",
                    "L34",
                    "L35",
                    "L36",
                    "L37",
                    "L39",
                    "L43",
                    "L45",
                    "L46",
                    "L47",
                    "L48",
                    "L49",
                    "L51",
                    "L52",
                    "L62",
                    "L63",
                    "L64",
                    "L65",
                    "L67",
                    "L71",
                    "L72",
                    "L77",
                    "L81",
                    "LAB01",
                    "MERAKI_SITE",
                    "MPLS",
                    "NSX-T",
                    "SILVERPEAK",
                    "VERSA_SITE",
                    "VIPTELA",
                ],
                "start": "2021-07-02T06:00:00.930000Z",
                "change": "2022-03-25T14:35:48.277000Z",
                "errors": [
                    {"count": 2, "error_type": "ABMapResultError"},
                    {"count": 5, "error_type": "ABParseError"},
                    {"count": 3, "error_type": "ABTaskMapResultError"},
                    {"count": 1, "error_type": "ABAmbiguousCommand"},
                    {"count": 7, "error_type": "ABCmdAuthFail"},
                    {"count": 6, "error_type": "ABCommandTimeout"},
                    {"count": 1, "error_type": "ABNoConfig"},
                    {"count": 1, "error_type": "ABParseBadConfigError"},
                    {"count": 1, "error_type": "ABTaskMapResultBadConfigError"},
                    {"count": 1, "error_type": "ABWorkerAuthError"},
                ],
                "locked": False,
                "status": "loaded",
                "loading": False,
                "version": "6.3.0-13",
                "user_count": 2324,
                "loaded_size": 170856074,
                "snapshot_id": "12dd8c61-129c-431a-b98b-4c9211571f89",
                "from_archive": True,
                "finish_status": "loaded",
                "unloaded_size": 26914884,
                "initial_version": "4.4.3+2",
                "interface_count": 9608,
                "total_dev_count": 729,
                "creator_username": None,
                "device_added_count": 0,
                "licensed_dev_count": 720,
                "device_removed_count": 0,
                "disabled_graph_cache": False,
                "interface_edge_count": 534,
                "interface_active_count": 6379,
                "disabled_historical_data": False,
                "disabled_intent_verification": False,
            },
            last_updated=timezone.now(),
        )
        sync = IPFabricSync.objects.create(
            name="ingest",
            status="new",
            snapshot_data=snapshot,
            update_custom_fields=True,
            parameters={
                "ipam.vrf": False,
                "dcim.site": True,
                "ipam.vlan": False,
                "groups": [],
                "dcim.device": True,
                "ipam.prefix": False,
                "dcim.platform": True,
                "dcim.interface": False,
                "ipam.ipaddress": False,
                "dcim.devicerole": True,
                "dcim.devicetype": True,
                "dcim.manufacturer": True,
                "dcim.virtualchassis": False,
            },
        )

        ingestion = IPFabricIngestion.objects.create(sync=sync)

        runner = IPFabricSyncRunner(
            settings={
                "dcim.site": True,
                "groups": [],
                "dcim.device": True,
                "dcim.platform": True,
                "dcim.interface": False,
                "dcim.devicerole": True,
                "dcim.devicetype": True,
                "dcim.manufacturer": True,
                "dcim.virtualchassis": True,
                "snapshot_id": "12dd8c61-129c-431a-b98b-4c9211571f89",
            },
            sync=sync,
            ingestion=ingestion,
        )
        # Need to monkeypatch since we are not in active Branch (different schema)
        # Using default schema "default" here since we are in "test_netbox" DB
        runner.get_db_connection_name = lambda: DEFAULT_DB_ALIAS

        site_data = {
            "siteName": "MPLS",
            "devicesCount": 1,
            "usersCount": 2,
            "stpDCount": 0,
            "switchesCount": 0,
            "vlanCount": 1,
            "rDCount": 0,
            "routersCount": 0,
            "networksCount": 6,
        }

        self.site = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="site", data=site_data
            ),
            cf=sync.update_custom_fields,
        )

        device_data = {
            "id": "961251111",
            "configReg": "0x0",
            "devType": "router",
            "family": "ios",
            "hostname": "L21PE152",
            "hostnameOriginal": None,
            "hostnameProcessed": None,
            "domain": None,
            "fqdn": None,
            "icon": None,
            "image": "unix:/opt/unetlab/addons/iol/bin/i86bi-linux-l3-adventerprisek9-15.2",
            "objectId": None,
            "taskKey": "fb67e3b4-5e48-4e52-b000-56cb187f2852",
            "loginIp": "10.21.254.152",
            "loginType": "telnet",
            "loginPort": None,
            "mac": None,
            "memoryTotalBytes": 396008048,
            "memoryUsedBytes": 72264172,
            "memoryUtilization": 18.25,
            "model": "",
            "platform": "i86bi_linux",
            "processor": "Intel-x86",
            "rd": "3",
            "reload": "reload at 0",
            "siteName": "MPLS",
            "sn": "a15ff98",
            "snHw": "a15ff98",
            "stpDomain": None,
            "uptime": 7254180,
            "vendor": "cisco",
            "version": "15.2(4)M1",
            "virtual_chassis": None,
            "slug": None,
        }

        self.mf_obj = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="manufacturer", data=device_data
            ),
            cf=sync.update_custom_fields,
        )
        self.dt_obj = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="devicetype", data=device_data
            ),
            cf=sync.update_custom_fields,
        )
        self.platform = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="platform", data=device_data
            ),
            cf=sync.update_custom_fields,
        )
        self.role = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="devicerole", data=device_data
            ),
            cf=sync.update_custom_fields,
        )
        self.device_object = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="device", data=device_data
            ),
            cf=sync.update_custom_fields,
        )

    def test_transform_map(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        self.assertEqual(site_transform_map.name, "Site Transform Map")
        self.assertIsNotNone(site_transform_map.source_endpoint)
        self.assertEqual(
            site_transform_map.source_endpoint.endpoint, "/inventory/sites/overview"
        )
        self.assertEqual(
            site_transform_map.target_model,
            ContentType.objects.filter(app_label="dcim", model="site")[0],
        )

    def test_transform_field(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        site_slug_field = IPFabricTransformField.objects.get(
            source_field="siteName",
            target_field="slug",
            transform_map=site_transform_map,
        )
        self.assertEqual(site_slug_field.source_field, "siteName")
        self.assertEqual(site_slug_field.target_field, "slug")
        self.assertEqual(site_slug_field.template, "{{ object.siteName | slugify }}")
        self.assertEqual(site_slug_field.transform_map, site_transform_map)
        site_name_field = IPFabricTransformField.objects.get(
            source_field="siteName",
            target_field="name",
            transform_map=site_transform_map,
        )
        self.assertEqual(site_name_field.source_field, "siteName")
        self.assertEqual(site_name_field.target_field, "name")
        self.assertEqual(site_name_field.template, "")
        self.assertEqual(site_name_field.transform_map, site_transform_map)

    def test_transform_map_serialization(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        data = serializers.serialize("json", [site_transform_map])
        data = json.loads(data)[0]
        test_data = {
            "model": "ipfabric_netbox.ipfabrictransformmap",
            "pk": site_transform_map.pk,
            "fields": {
                "name": "Site Transform Map",
                "source_endpoint": site_transform_map.source_endpoint.pk,
                "target_model": ContentType.objects.get(
                    app_label="dcim", model="site"
                ).pk,
            },
        }
        new_data = copy.deepcopy(data)
        new_data.pop("fields")
        new_fields = {}
        for k in test_data["fields"]:
            new_fields[k] = data["fields"][k]
        new_data["fields"] = new_fields
        self.assertDictEqual(test_data, new_data)

    def test_transform_field_serialization(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        site_slug_field = IPFabricTransformField.objects.get(
            source_field="siteName", target_field="slug"
        )
        data = serializers.serialize("json", [site_slug_field])
        data = json.loads(data)[0]
        test_data = {
            "model": "ipfabric_netbox.ipfabrictransformfield",
            "pk": site_slug_field.pk,
            "fields": {
                "source_field": "siteName",
                "target_field": "slug",
                "template": "{{ object.siteName | slugify }}",
                "transform_map": site_transform_map.pk,
            },
        }
        new_data = copy.deepcopy(data)
        new_data.pop("fields")
        new_data["fields"] = {k: data["fields"][k] for k in test_data["fields"]}
        self.assertDictEqual(test_data, new_data)

    def test_update_or_create_instance_site(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        data = {
            "siteName": "Site 1",
            "devicesCount": 1,
            "usersCount": 2,
            "stpDCount": 0,
            "switchesCount": 0,
            "vlanCount": 1,
            "rDCount": 0,
            "routersCount": 0,
            "networksCount": 6,
        }
        context = site_transform_map.get_context(data)
        object = site_transform_map.update_or_create_instance(context)
        self.assertEqual(object.name, "Site 1")
        self.assertEqual(object.slug, "site-1")

    def test_update_or_create_instance_device(self):
        device_object = Device.objects.first()

        self.assertEqual(device_object.name, "L21PE152")
        self.assertEqual(device_object.serial, "a15ff98")
        self.assertEqual(device_object.platform, self.platform)
        self.assertEqual(device_object.role, self.role)
        self.assertEqual(device_object.device_type, self.dt_obj)
        self.assertEqual(device_object.device_type.manufacturer, self.mf_obj)
        self.assertEqual(device_object.site, self.site)
        self.assertEqual(device_object.status, "active")

    def test_alter_transform_field_template(self):
        sync = IPFabricSync.objects.get(name="ingest")

        runner = IPFabricSyncRunner(
            settings={
                "dcim.site": True,
                "groups": [],
                "dcim.device": True,
                "dcim.platform": True,
                "dcim.interface": False,
                "dcim.devicerole": True,
                "dcim.devicetype": True,
                "dcim.manufacturer": True,
                "dcim.virtualchassis": True,
                "snapshot_id": "12dd8c61-129c-431a-b98b-4c9211571f89",
            },
            sync=sync,
        )
        # Need to monkeypatch since we are not in active Branch (different schema)
        # Using default schema "default" here since we are in "test_netbox" DB
        runner.get_db_connection_name = lambda: DEFAULT_DB_ALIAS

        device_data = {
            "id": "961251111",
            "configReg": "0x0",
            "devType": "router",
            "family": "ios",
            "hostname": "L21PE152",
            "hostnameOriginal": None,
            "hostnameProcessed": None,
            "domain": None,
            "fqdn": None,
            "icon": None,
            "image": "unix:/opt/unetlab/addons/iol/bin/i86bi-linux-l3-adventerprisek9-15.2",
            "objectId": None,
            "taskKey": "fb67e3b4-5e48-4e52-b000-56cb187f2852",
            "loginIp": "10.21.254.152",
            "loginType": "telnet",
            "loginPort": None,
            "mac": None,
            "memoryTotalBytes": 396008048,
            "memoryUsedBytes": 72264172,
            "memoryUtilization": 18.25,
            "model": "",
            "platform": "i86bi_linux",
            "processor": "Intel-x86",
            "rd": "3",
            "reload": "reload at 0",
            "siteName": "MPLS",
            "sn": "a15ff98",
            "snHw": "a15ff98",
            "stpDomain": None,
            "uptime": 7254180,
            "vendor": "cisco",
            "version": "15.2(4)M1",
            "virtual_chassis": None,
            "slug": None,
        }

        device_endpoint = IPFabricEndpoint.objects.get(endpoint="/inventory/devices")
        transform_field = IPFabricTransformField.objects.get(
            source_field="hostname",
            target_field="name",
            transform_map=IPFabricTransformMap.objects.get(
                source_endpoint=device_endpoint,
                target_model=ContentType.objects.get(app_label="dcim", model="device"),
            ),
        )
        transform_field.template = "{{ object.hostname }} - test"
        transform_field.save()
        device_object = runner.sync_item(
            record=runner.create_new_data_record(
                app="dcim", model="device", data=device_data
            ),
            cf=sync.update_custom_fields,
        )
        self.assertEqual(device_object.name, "L21PE152 - test")

    # ------------------------------------------------------------------
    # get_model_hierarchy tests
    # ------------------------------------------------------------------

    def test_group_override_does_not_raise_false_circular_dependency(self):
        """
        Default maps (no group):
            A: parts        -> inventory_item   (no parents)
            B: interfaces   -> interface        (parents=[A])
            C: interfaces   -> macaddress       (parents=[A])
            D: managed_ip   -> ipaddress        (parents=[B, C])

          Group G overrides B, C, D with clones B', C', D'.
          The clones are created with parents pointing to the *original* default
          A/B/C IDs (as a UI clone would do).

        The hierarchy should be resolved correctly with no error.
        """
        A = self._default_map(
            "Default A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Default B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        C = self._default_map("Default C (ifaces->mac)", self.ep_ifaces, self.ct_mac)
        D = self._default_map(
            "Default D (mgdip->ipaddr)", self.ep_managed_ip, self.ct_ipaddress
        )

        # Wire default hierarchy: B->A, C->A, D->B+C
        B.parents.add(A)
        C.parents.add(A)
        D.parents.add(B, C)

        group = IPFabricTransformMapGroup.objects.create(name="Override Group")

        # Clone B, C, D into the group; parents intentionally still point to *default* IDs
        B_prime = self._group_map(
            "Group B' (ifaces->iface)", self.ep_ifaces, self.ct_interface, group
        )
        C_prime = self._group_map(
            "Group C' (ifaces->mac)", self.ep_ifaces, self.ct_mac, group
        )
        D_prime = self._group_map(
            "Group D' (mgdip->ipaddr)", self.ep_managed_ip, self.ct_ipaddress, group
        )

        # Clones point to the ORIGINAL default IDs (this is what a manual clone produces)
        B_prime.parents.add(A)  # A is not overridden — still valid
        C_prime.parents.add(A)
        D_prime.parents.add(B, C)  # B and C are the *default* IDs, not B'/C'

        # Should NOT raise
        result = IPFabricSync.get_model_hierarchy(group_ids=[group.id])

        active_ids = {tm.id for tm in result}

        # Active set must contain the group overrides, not the originals
        self.assertIn(B_prime.id, active_ids)
        self.assertIn(C_prime.id, active_ids)
        self.assertIn(D_prime.id, active_ids)
        self.assertNotIn(B.id, active_ids)
        self.assertNotIn(C.id, active_ids)
        self.assertNotIn(D.id, active_ids)

        # Dependency ordering: A before B', C'; B' and C' before D'
        idx = {tm.id: i for i, tm in enumerate(result)}
        self.assertLess(idx[A.id], idx[B_prime.id])
        self.assertLess(idx[A.id], idx[C_prime.id])
        self.assertLess(idx[B_prime.id], idx[D_prime.id])
        self.assertLess(idx[C_prime.id], idx[D_prime.id])

    def test_no_groups_default_hierarchy_unchanged(self):
        """Without any groups the existing default hierarchy must work as before."""
        A = self._default_map(
            "NoGrp A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "NoGrp B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        B.parents.add(A)

        result = IPFabricSync.get_model_hierarchy(group_ids=[])
        ids = [tm.id for tm in result]

        self.assertIn(A.id, ids)
        self.assertIn(B.id, ids)
        self.assertLess(ids.index(A.id), ids.index(B.id))

    def test_group_fully_overrides_chain_preserves_order(self):
        """
        When a group overrides every map in a chain the order must still be correct
        and no error raised.
        """
        A = self._default_map(
            "Chain A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Chain B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        B.parents.add(A)

        group = IPFabricTransformMapGroup.objects.create(name="Full Override Group")

        A_prime = self._group_map(
            "Chain A' (parts->invitem)", self.ep_parts, self.ct_inventory_item, group
        )
        B_prime = self._group_map(
            "Chain B' (ifaces->iface)", self.ep_ifaces, self.ct_interface, group
        )

        # B' parent is the original A (not A') — the typical cloning scenario
        B_prime.parents.add(A)

        result = IPFabricSync.get_model_hierarchy(group_ids=[group.id])
        idx = {tm.id: i for i, tm in enumerate(result)}

        # Both overrides must be present; originals must not
        self.assertIn(A_prime.id, idx)
        self.assertIn(B_prime.id, idx)
        self.assertNotIn(A.id, idx)
        self.assertNotIn(B.id, idx)
        # Order preserved: A' before B'
        self.assertLess(idx[A_prime.id], idx[B_prime.id])

    def test_true_circular_dependency_still_raises(self):
        """
        A genuine cycle in default maps must still raise ValidationError.
        The cycle is injected directly via the M2M through table to bypass clean().
        """
        A = self._default_map(
            "Cycle A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Cycle B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )

        # A -> B -> A  (cycle)
        IPFabricTransformMap.parents.through.objects.create(
            from_ipfabrictransformmap=A,
            to_ipfabrictransformmap=B,
        )
        IPFabricTransformMap.parents.through.objects.create(
            from_ipfabrictransformmap=B,
            to_ipfabrictransformmap=A,
        )

        with self.assertRaises(ValidationError) as ctx:
            IPFabricSync.get_model_hierarchy(group_ids=[])

        self.assertIn("Circular dependency", str(ctx.exception))

    def test_m2m_signal_raises_on_cycle(self):
        """
        validate_circular_dependency_on_m2m_change fires on 'pre_add' and must
        raise ValidationError when parents.add() would close a cycle.
        This is a distinct code path from clean() — it catches programmatic /
        API mutations that bypass form validation entirely.

        Chain before the bad add:  A -> B -> C  (valid, no cycle)
        After A.parents.add(C):    A -> B -> C -> A  (cycle)
        """
        A = self._default_map(
            "Signal Cycle A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Signal Cycle B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        C = self._default_map(
            "Signal Cycle C (ifaces->mac)", self.ep_ifaces, self.ct_mac
        )

        # Build a valid three-node chain A -> B -> C
        B.parents.add(A)
        C.parents.add(B)

        # Adding C as a parent of A would close the cycle A -> B -> C -> A
        with self.assertRaises(ValidationError) as ctx:
            A.parents.add(C)

        self.assertIn("circular dependency detected", str(ctx.exception).lower())

    def test_m2m_signal_allows_valid_parent_add(self):
        """
        validate_circular_dependency_on_m2m_change must NOT raise when the
        new parent relationship is acyclic.
        """
        A = self._default_map(
            "Signal Valid A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Signal Valid B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        C = self._default_map(
            "Signal Valid C (ifaces->mac)", self.ep_ifaces, self.ct_mac
        )

        # B depends on A — valid
        B.parents.add(A)

        # C depends on B — still valid, no cycle introduced
        C.parents.add(B)  # must not raise

        self.assertIn(B, list(C.parents.all()))


class IPFabricFilterCombinationTestCase(TestCase):
    """Test cases for recursive filter merging with nested and/or structures."""

    @classmethod
    def setUpTestData(cls):
        # Create source
        cls.source = IPFabricSource.objects.create(
            name="Test Source",
            url="https://test.ipfabric.io",
            status="active",
            parameters={"auth": "test_token", "verify": False},
        )
        # Create snapshot
        cls.snapshot = IPFabricSnapshot.objects.create(
            name="Test Snapshot",
            source=cls.source,
            snapshot_id="test-snapshot-123",
            status=IPFabricSnapshotStatusModelChoices.STATUS_LOADED,
            data={"sites": ["Site1", "Site2"]},
        )
        # Create sync
        cls.sync = IPFabricSync.objects.create(
            name="Test Sync",
            snapshot_data=cls.snapshot,
            status="new",
        )
        # Get or create endpoint
        cls.endpoint, _ = IPFabricEndpoint.objects.get_or_create(
            endpoint="/inventory/devices",
            defaults={"name": "Devices Endpoint"},
        )

    def test_combine_filters_merges_same_nested_or_structure(self):
        """Test that filters with same nested 'or' structure merge correctly."""
        # Create two AND-type filters
        filter1 = IPFabricFilter.objects.create(
            name="Filter with router1",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        filter2 = IPFabricFilter.objects.create(
            name="Filter with router2",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter2.endpoints.add(self.endpoint)
        filter2.syncs.add(self.sync)

        # Create expressions with same nested structure
        expr1 = IPFabricFilterExpression.objects.create(
            name="Expression router1",
            expression=[{"or": [{"hostname": ["like", "router1"]}]}],
        )
        expr1.filters.add(filter1)

        expr2 = IPFabricFilterExpression.objects.create(
            name="Expression router2",
            expression=[{"or": [{"hostname": ["like", "router2"]}]}],
        )
        expr2.filters.add(filter2)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have merged into single nested structure
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])
        self.assertEqual(len(result["and"][0]["or"]), 2)

        # Verify both hostnames are present
        hostnames = [item["hostname"][1] for item in result["and"][0]["or"]]
        self.assertIn("router1", hostnames)
        self.assertIn("router2", hostnames)

    def test_combine_filters_deeply_nested_structures(self):
        """Test that deeply nested structures (3+ levels) merge correctly."""
        # Create two AND-type filters with 3-level nesting
        filter1 = IPFabricFilter.objects.create(
            name="Deep filter cisco",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        filter2 = IPFabricFilter.objects.create(
            name="Deep filter juniper",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter2.endpoints.add(self.endpoint)
        filter2.syncs.add(self.sync)

        # Create deeply nested expressions
        expr1 = IPFabricFilterExpression.objects.create(
            name="Deep expression cisco",
            expression=[{"or": [{"and": [{"or": [{"vendor": ["eq", "cisco"]}]}]}]}],
        )
        expr1.filters.add(filter1)

        expr2 = IPFabricFilterExpression.objects.create(
            name="Deep expression juniper",
            expression=[{"or": [{"and": [{"or": [{"vendor": ["eq", "juniper"]}]}]}]}],
        )
        expr2.filters.add(filter2)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have merged at all levels
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])
        self.assertEqual(len(result["and"][0]["or"]), 1)
        self.assertIn("and", result["and"][0]["or"][0])
        self.assertEqual(len(result["and"][0]["or"][0]["and"]), 1)
        self.assertIn("or", result["and"][0]["or"][0]["and"][0])

        # Verify both vendors are in innermost or array
        innermost_or = result["and"][0]["or"][0]["and"][0]["or"]
        self.assertEqual(len(innermost_or), 2)
        vendors = [item["vendor"][1] for item in innermost_or]
        self.assertIn("cisco", vendors)
        self.assertIn("juniper", vendors)

    def test_combine_filters_different_structures_no_merge(self):
        """Test that filters with different structures don't merge."""
        # Create two AND-type filters with different nested structures
        filter1 = IPFabricFilter.objects.create(
            name="Filter with or structure",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        filter2 = IPFabricFilter.objects.create(
            name="Filter with and structure",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter2.endpoints.add(self.endpoint)
        filter2.syncs.add(self.sync)

        # Create expressions with different structures
        expr1 = IPFabricFilterExpression.objects.create(
            name="Expression with or",
            expression=[{"or": [{"hostname": ["like", "router1"]}]}],
        )
        expr1.filters.add(filter1)

        expr2 = IPFabricFilterExpression.objects.create(
            name="Expression with and",
            expression=[{"and": [{"hostname": ["like", "router2"]}]}],
        )
        expr2.filters.add(filter2)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have two separate structures (not merged)
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 2)

        # One should have 'or', the other should have 'and'
        structures = [list(item.keys()) for item in result["and"]]
        self.assertIn(["or"], structures)
        self.assertIn(["and"], structures)

    def test_combine_filters_multiple_filters_same_structure(self):
        """Test that 3+ filters with same structure all merge correctly."""
        # Create three AND-type filters with same nested structure
        for i in range(1, 4):
            filter_obj = IPFabricFilter.objects.create(
                name=f"Filter with router{i}",
                filter_type=IPFabricFilterTypeChoices.AND,
            )
            filter_obj.endpoints.add(self.endpoint)
            filter_obj.syncs.add(self.sync)

            expr = IPFabricFilterExpression.objects.create(
                name=f"Expression router{i}",
                expression=[{"or": [{"hostname": ["like", f"router{i}"]}]}],
            )
            expr.filters.add(filter_obj)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have merged all three into single nested structure
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])
        self.assertEqual(len(result["and"][0]["or"]), 3)

        # Verify all three hostnames are present
        hostnames = [item["hostname"][1] for item in result["and"][0]["or"]]
        self.assertIn("router1", hostnames)
        self.assertIn("router2", hostnames)
        self.assertIn("router3", hostnames)


class SyncLoggingTestCase(TestCase):
    """Tests for the SyncLogging utility class."""

    def setUp(self):
        self.logger = SyncLogging(key_prefix="test_sync", job=999)

    def test_initial_log_data_structure(self):
        """log_data must contain a single statistics key (no phase split)."""
        self.assertIn("logs", self.logger.log_data)
        self.assertIn("statistics", self.logger.log_data)
        self.assertNotIn("sync_statistics", self.logger.log_data)
        self.assertNotIn("merge_statistics", self.logger.log_data)
        self.assertEqual(self.logger.log_data["statistics"], {})

    def test_init_statistics(self):
        """init_statistics creates entry under statistics with current=0."""
        self.logger.init_statistics("dcim.device", total=10)
        self.assertIn("dcim.device", self.logger.log_data["statistics"])
        self.assertEqual(
            self.logger.log_data["statistics"]["dcim.device"],
            {"current": 0, "total": 10},
        )

    def test_init_statistics_does_not_overwrite(self):
        """init_statistics is idempotent — a second call does not reset current."""
        self.logger.init_statistics("dcim.device", total=10)
        self.logger.increment_statistics("dcim.device")
        self.logger.init_statistics("dcim.device", total=99)  # should be a no-op
        self.assertEqual(
            self.logger.log_data["statistics"]["dcim.device"],
            {"current": 1, "total": 10},
        )

    def test_increment_statistics(self):
        """increment_statistics increments current counter."""
        self.logger.init_statistics("dcim.device", total=5)
        self.logger.increment_statistics("dcim.device")
        self.logger.increment_statistics("dcim.device")
        self.assertEqual(
            self.logger.log_data["statistics"]["dcim.device"]["current"], 2
        )

    def test_increment_statistics_auto_creates_entry(self):
        """increment_statistics creates the entry if not pre-initialised."""
        self.logger.increment_statistics("ipam.prefix")
        self.assertIn("ipam.prefix", self.logger.log_data["statistics"])
        self.assertEqual(
            self.logger.log_data["statistics"]["ipam.prefix"]["current"], 1
        )


class MergeBranchTestCase(TestCase):
    """Tests for the merge_branch() utility function."""

    def _make_ingestion(self, branch=None):
        source = IPFabricSource.objects.create(
            name="Merge Test Source",
            url="https://merge.test.local",
            parameters={"auth": "token", "verify": True},
        )
        snapshot = IPFabricSnapshot.objects.create(
            name="Merge Test Snapshot",
            source=source,
            snapshot_id="merge-snap-1",
            status="loaded",
            data={},
        )
        user = get_user_model().objects.create_user(username="merge-test-user")
        sync = IPFabricSync.objects.create(
            name="Merge Test Sync",
            snapshot_data=snapshot,
            parameters={},
            user=user,
        )
        ingestion = IPFabricIngestion.objects.create(sync=sync, branch=branch)
        sync.logger = SyncLogging(key_prefix="test_merge", job=sync.pk)
        return ingestion

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.BranchEvent.objects")
    @patch("ipfabric_netbox.utilities.merge.post_merge")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    def test_no_changes_aborts_early(
        self, mock_post_save, mock_post_merge, mock_branch_event, mock_branch_objects
    ):
        """merge_branch returns early without touching the branch when there are no changes."""
        branch = MagicMock()
        branch.ready = True
        branch.get_unmerged_changes.return_value.order_by.return_value.count.return_value = (
            0
        )
        ingestion = MagicMock()
        ingestion.branch = branch

        merge_branch(ingestion)

        mock_branch_objects.filter.assert_not_called()
        mock_post_merge.send.assert_not_called()
        mock_branch_event.create.assert_not_called()

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.BranchEvent.objects")
    @patch("ipfabric_netbox.utilities.merge.post_merge")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    @patch("ipfabric_netbox.utilities.merge.connection")
    @patch("ipfabric_netbox.utilities.merge.get_merge_strategy")
    def test_successful_change_applied(
        self,
        mock_get_strategy,
        mock_connection,
        mock_post_save,
        mock_post_merge,
        mock_branch_event,
        mock_branch_objects,
    ):
        """A change that applies without error is committed and counted in merge_statistics."""
        mock_strategy = MagicMock()
        mock_get_strategy.return_value.return_value = mock_strategy
        mock_connection.savepoint.return_value = "sp1"

        change = MagicMock()
        change.pk = 42
        change.action = "create"
        change.changed_object_id = 1
        ct = MagicMock()
        ct.natural_key.return_value = ("dcim", "device")
        ct.model_class.return_value = MagicMock()
        change.changed_object_type = ct

        branch = MagicMock()
        branch.ready = True
        branch.schema_name = "branch_abc"
        branch.merge_strategy = None
        qs = MagicMock()
        qs.count.return_value = 1
        qs.values_list.return_value = [("dcim", "device")]
        qs.__iter__ = MagicMock(return_value=iter([change]))
        branch.get_unmerged_changes.return_value.order_by.return_value = qs

        ingestion = MagicMock()
        ingestion.branch = branch
        sync_logger = SyncLogging(key_prefix="test", job=1)
        ingestion.sync.user = MagicMock()

        merge_branch(ingestion, sync_logger=sync_logger)

        mock_connection.savepoint_commit.assert_called_once_with("sp1")
        mock_connection.savepoint_rollback.assert_not_called()
        self.assertEqual(
            sync_logger.log_data["statistics"]["dcim.device"]["current"], 1
        )
        mock_branch_event.create.assert_called_once()
        mock_post_merge.send.assert_called_once()

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.BranchEvent.objects")
    @patch("ipfabric_netbox.utilities.merge.post_merge")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    @patch("ipfabric_netbox.utilities.merge.connection")
    @patch("ipfabric_netbox.utilities.merge.get_merge_strategy")
    @patch("ipfabric_netbox.utilities.merge.IPFabricIngestionIssue")
    def test_failed_change_creates_issue_and_continues(
        self,
        mock_issue_class,
        mock_get_strategy,
        mock_connection,
        mock_post_save,
        mock_post_merge,
        mock_branch_event,
        mock_branch_objects,
    ):
        """A change that raises is rolled back, recorded as IPFabricIngestionIssue, and
        the loop continues to apply the next change."""
        mock_strategy = MagicMock()
        mock_get_strategy.return_value.return_value = mock_strategy
        mock_connection.savepoint.side_effect = ["sp1", "sp2"]

        def make_change(pk, action, should_fail):
            change = MagicMock()
            change.pk = pk
            change.action = action
            change.changed_object_id = pk
            ct = MagicMock()
            ct.natural_key.return_value = ("dcim", "device")
            ct.model_class.return_value = MagicMock()
            change.changed_object_type = ct
            if should_fail:
                change.apply.side_effect = ValueError("boom")
            return change

        bad_change = make_change(1, "create", should_fail=True)
        good_change = make_change(2, "update", should_fail=False)

        branch = MagicMock()
        branch.ready = True
        branch.schema_name = "branch_xyz"
        branch.merge_strategy = None
        qs = MagicMock()
        qs.count.return_value = 2
        qs.values_list.return_value = [("dcim", "device"), ("dcim", "device")]
        qs.__iter__ = MagicMock(return_value=iter([bad_change, good_change]))
        branch.get_unmerged_changes.return_value.order_by.return_value = qs

        ingestion = MagicMock()
        ingestion.branch = branch
        sync_logger = SyncLogging(key_prefix="fail_test", job=1)

        merge_branch(ingestion, sync_logger=sync_logger)

        # Bad change: savepoint rolled back
        mock_connection.savepoint_rollback.assert_called_once_with("sp1")
        # Good change: savepoint committed
        mock_connection.savepoint_commit.assert_called_once_with("sp2")

        # One issue created for the bad change — assert on the create() call kwargs
        mock_issue_class.objects.create.assert_called_once()
        call_kwargs = mock_issue_class.objects.create.call_args.kwargs
        self.assertEqual(call_kwargs["phase"], IPFabricIngestionPhaseChoices.MERGE)
        self.assertEqual(call_kwargs["model"], "dcim.device")
        self.assertIn("boom", call_kwargs["message"])
        self.assertEqual(call_kwargs["exception"], "ValueError")

        # Merge still completed (BranchEvent + post_merge fired)
        mock_branch_event.create.assert_called_once()
        mock_post_merge.send.assert_called_once()

        # Progress: 1 success out of 2 total
        self.assertEqual(
            sync_logger.log_data["statistics"]["dcim.device"]["current"], 1
        )

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    def test_signal_disconnected_on_exception(
        self, mock_post_save, mock_branch_objects
    ):
        """record_applied_change signal is disconnected even when an unexpected
        exception escapes the loop (e.g. from branch status update)."""
        branch = MagicMock()
        branch.ready = True
        branch.schema_name = "branch_err"
        qs = MagicMock()
        qs.count.return_value = 1
        qs.values_list.return_value = [("dcim", "site")]
        # Make iteration itself raise to simulate an unexpected error
        qs.__iter__ = MagicMock(side_effect=RuntimeError("unexpected"))
        branch.get_unmerged_changes.return_value.order_by.return_value = qs

        ingestion = MagicMock()
        ingestion.branch = branch

        with self.assertRaises(RuntimeError):
            merge_branch(ingestion)

        # disconnect must have been called despite the exception
        mock_post_save.disconnect.assert_called_once()
