import copy
import json
from unittest.mock import MagicMock
from unittest.mock import patch

from dcim.models import Device
from django.contrib.auth import get_user_model
from django.contrib.contenttypes.models import ContentType
from django.core import serializers
from django.core.exceptions import ValidationError
from django.db import DEFAULT_DB_ALIAS
from django.test import TestCase
from django.utils import timezone
from extras.models import CustomField

from ipfabric_netbox.choices import IPFabricFilterTypeChoices
from ipfabric_netbox.choices import IPFabricIngestionPhaseChoices
from ipfabric_netbox.choices import IPFabricSnapshotStatusModelChoices
from ipfabric_netbox.models import IPFabricEndpoint
from ipfabric_netbox.models import IPFabricFilter
from ipfabric_netbox.models import IPFabricFilterExpression
from ipfabric_netbox.models import IPFabricIngestion
from ipfabric_netbox.models import IPFabricRelationshipField
from ipfabric_netbox.models import IPFabricSnapshot
from ipfabric_netbox.models import IPFabricSource
from ipfabric_netbox.models import IPFabricSync
from ipfabric_netbox.models import IPFabricTransformField
from ipfabric_netbox.models import IPFabricTransformMap
from ipfabric_netbox.models import IPFabricTransformMapGroup
from ipfabric_netbox.utilities.ipfutils import IPFabricSyncRunner
from ipfabric_netbox.utilities.logging import SyncLogging
from ipfabric_netbox.utilities.merge import merge_branch


class IPFabricTransformMapModelTestCase(TestCase):
    @classmethod
    def setUpTestData(cls):
        # Endpoints and content types used by the get_model_hierarchy tests below.
        cls.ep_parts = IPFabricEndpoint.objects.get(endpoint="/inventory/part-numbers")
        cls.ep_ifaces = IPFabricEndpoint.objects.get(endpoint="/inventory/interfaces")
        cls.ep_managed_ip = IPFabricEndpoint.objects.get(
            endpoint="/technology/addressing/managed-ip/ipv4"
        )
        cls.ct_inventory_item = ContentType.objects.get(
            app_label="dcim", model="inventoryitem"
        )
        cls.ct_interface = ContentType.objects.get(app_label="dcim", model="interface")
        cls.ct_mac = ContentType.objects.get(app_label="dcim", model="macaddress")
        cls.ct_ipaddress = ContentType.objects.get(app_label="ipam", model="ipaddress")

    # ------------------------------------------------------------------
    # Helpers used by the get_model_hierarchy tests
    # ------------------------------------------------------------------

    def _default_map(self, name, endpoint, ct):
        """Create a default (no-group) transform map."""
        return IPFabricTransformMap.objects.create(
            name=name,
            source_endpoint=endpoint,
            target_model=ct,
            group=None,
        )

    def _group_map(self, name, endpoint, ct, group):
        """Create a group-owned transform map."""
        return IPFabricTransformMap.objects.create(
            name=name,
            source_endpoint=endpoint,
            target_model=ct,
            group=group,
        )

    def setUp(self):
        source = IPFabricSource.objects.create(
            name="test",
            url="https://localhost",
            status="new",
            parameters={"auth": "token123", "verify": True},
        )
        snapshot = IPFabricSnapshot.objects.create(
            name="S01 - Day 2 - IPF Lab - 02-Jul-21 06:29:16 - 12dd8c61-129c-431a-b98b-4c9211571f89",
            source=source,
            snapshot_id="12dd8c61-129c-431a-b98b-4c9211571f89",
            status=IPFabricSnapshotStatusModelChoices.STATUS_LOADED,
            data={
                "end": "2021-07-02T06:29:16.311000Z",
                "name": "S01 - Day 2 - IPF Lab",
                "note": "Multi-Environment containing:\\nAWS, Azure, ACI, NSX-T, Viptela, Versa, SilverPeak, Meraki\\n\\nSite 48 - devices added, NTP issue\\nSite 68 &38 - NTP update\\nSite 38 - resiliency affected (no ospfx2 - no L1 link x1) + passive interfaces FIXED / NTP partial update\\n?E2E: 38 - 66 - migration HTTP to HTTPS\\n?Site 66 - FW bypass E2E\\nVRRP improvements (LAB1 / L52)",
                "sites": [
                    "35COLO",
                    "35HEADOFFICE",
                    "35PRODUCTION",
                    "35SALES",
                    "38 Pilsen DR",
                    "66 Ostrava DC",
                    "68 Pardubice Distribution",
                    "ACI",
                    "AWS_SITE",
                    "AZURE",
                    "HWLAB",
                    "L31",
                    "L33",
                    "L34",
                    "L35",
                    "L36",
                    "L37",
                    "L39",
                    "L43",
                    "L45",
                    "L46",
                    "L47",
                    "L48",
                    "L49",
                    "L51",
                    "L52",
                    "L62",
                    "L63",
                    "L64",
                    "L65",
                    "L67",
                    "L71",
                    "L72",
                    "L77",
                    "L81",
                    "LAB01",
                    "MERAKI_SITE",
                    "MPLS",
                    "NSX-T",
                    "SILVERPEAK",
                    "VERSA_SITE",
                    "VIPTELA",
                ],
                "start": "2021-07-02T06:00:00.930000Z",
                "change": "2022-03-25T14:35:48.277000Z",
                "errors": [
                    {"count": 2, "error_type": "ABMapResultError"},
                    {"count": 5, "error_type": "ABParseError"},
                    {"count": 3, "error_type": "ABTaskMapResultError"},
                    {"count": 1, "error_type": "ABAmbiguousCommand"},
                    {"count": 7, "error_type": "ABCmdAuthFail"},
                    {"count": 6, "error_type": "ABCommandTimeout"},
                    {"count": 1, "error_type": "ABNoConfig"},
                    {"count": 1, "error_type": "ABParseBadConfigError"},
                    {"count": 1, "error_type": "ABTaskMapResultBadConfigError"},
                    {"count": 1, "error_type": "ABWorkerAuthError"},
                ],
                "locked": False,
                "status": "loaded",
                "loading": False,
                "version": "6.3.0-13",
                "user_count": 2324,
                "loaded_size": 170856074,
                "snapshot_id": "12dd8c61-129c-431a-b98b-4c9211571f89",
                "from_archive": True,
                "finish_status": "loaded",
                "unloaded_size": 26914884,
                "initial_version": "4.4.3+2",
                "interface_count": 9608,
                "total_dev_count": 729,
                "creator_username": None,
                "device_added_count": 0,
                "licensed_dev_count": 720,
                "device_removed_count": 0,
                "disabled_graph_cache": False,
                "interface_edge_count": 534,
                "interface_active_count": 6379,
                "disabled_historical_data": False,
                "disabled_intent_verification": False,
            },
            last_updated=timezone.now(),
        )
        sync = IPFabricSync.objects.create(
            name="ingest",
            status="new",
            snapshot_data=snapshot,
            parameters={
                "ipam.vrf": False,
                "dcim.site": True,
                "ipam.vlan": False,
                "groups": [],
                "dcim.device": True,
                "ipam.prefix": False,
                "dcim.platform": True,
                "dcim.interface": False,
                "ipam.ipaddress": False,
                "dcim.devicerole": True,
                "dcim.devicetype": True,
                "dcim.manufacturer": True,
                "dcim.virtualchassis": False,
            },
        )

        ingestion = IPFabricIngestion.objects.create(sync=sync)

        runner = IPFabricSyncRunner(
            settings={
                "dcim.site": True,
                "groups": [],
                "dcim.device": True,
                "dcim.platform": True,
                "dcim.interface": False,
                "dcim.devicerole": True,
                "dcim.devicetype": True,
                "dcim.manufacturer": True,
                "dcim.virtualchassis": True,
                "snapshot_id": "12dd8c61-129c-431a-b98b-4c9211571f89",
            },
            sync=sync,
            ingestion=ingestion,
        )
        # Need to monkeypatch since we are not in active Branch (different schema)
        # Using default schema "default" here since we are in "test_netbox" DB
        runner.get_db_connection_name = lambda: DEFAULT_DB_ALIAS

        site_data = {
            "siteName": "MPLS",
            "devicesCount": 1,
            "usersCount": 2,
            "stpDCount": 0,
            "switchesCount": 0,
            "vlanCount": 1,
            "rDCount": 0,
            "routersCount": 0,
            "networksCount": 6,
        }

        self.site = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="site", data=site_data
            ),
        )

        device_data = {
            "id": "961251111",
            "configReg": "0x0",
            "devType": "router",
            "family": "ios",
            "hostname": "L21PE152",
            "hostnameOriginal": None,
            "hostnameProcessed": None,
            "domain": None,
            "fqdn": None,
            "icon": None,
            "image": "unix:/opt/unetlab/addons/iol/bin/i86bi-linux-l3-adventerprisek9-15.2",
            "objectId": None,
            "taskKey": "fb67e3b4-5e48-4e52-b000-56cb187f2852",
            "loginIp": "10.21.254.152",
            "loginType": "telnet",
            "loginPort": None,
            "mac": None,
            "memoryTotalBytes": 396008048,
            "memoryUsedBytes": 72264172,
            "memoryUtilization": 18.25,
            "model": "",
            "platform": "i86bi_linux",
            "processor": "Intel-x86",
            "rd": "3",
            "reload": "reload at 0",
            "siteName": "MPLS",
            "sn": "a15ff98",
            "snHw": "a15ff98",
            "stpDomain": None,
            "uptime": 7254180,
            "vendor": "cisco",
            "version": "15.2(4)M1",
            "virtual_chassis": None,
            "slug": None,
        }

        self.mf_obj = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="manufacturer", data=device_data
            ),
        )
        self.dt_obj = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="devicetype", data=device_data
            ),
        )
        self.platform = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="platform", data=device_data
            ),
        )
        self.role = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="devicerole", data=device_data
            ),
        )
        self.device_object = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="device", data=device_data
            ),
        )

    def test_transform_map(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        self.assertEqual(site_transform_map.name, "Site Transform Map")
        self.assertIsNotNone(site_transform_map.source_endpoint)
        self.assertEqual(
            site_transform_map.source_endpoint.endpoint, "/inventory/sites/overview"
        )
        self.assertEqual(
            site_transform_map.target_model,
            ContentType.objects.filter(app_label="dcim", model="site")[0],
        )

    def test_transform_field(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        site_slug_field = IPFabricTransformField.objects.get(
            source_field="siteName",
            target_field="slug",
            transform_map=site_transform_map,
        )
        self.assertEqual(site_slug_field.source_field, "siteName")
        self.assertEqual(site_slug_field.target_field, "slug")
        self.assertEqual(site_slug_field.template, "{{ object.siteName | slugify }}")
        self.assertEqual(site_slug_field.transform_map, site_transform_map)
        site_name_field = IPFabricTransformField.objects.get(
            source_field="siteName",
            target_field="name",
            transform_map=site_transform_map,
        )
        self.assertEqual(site_name_field.source_field, "siteName")
        self.assertEqual(site_name_field.target_field, "name")
        self.assertEqual(site_name_field.template, "")
        self.assertEqual(site_name_field.transform_map, site_transform_map)

    def test_transform_map_serialization(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        data = serializers.serialize("json", [site_transform_map])
        data = json.loads(data)[0]
        test_data = {
            "model": "ipfabric_netbox.ipfabrictransformmap",
            "pk": site_transform_map.pk,
            "fields": {
                "name": "Site Transform Map",
                "source_endpoint": site_transform_map.source_endpoint.pk,
                "target_model": ContentType.objects.get(
                    app_label="dcim", model="site"
                ).pk,
            },
        }
        new_data = copy.deepcopy(data)
        new_data.pop("fields")
        new_fields = {}
        for k in test_data["fields"]:
            new_fields[k] = data["fields"][k]
        new_data["fields"] = new_fields
        self.assertDictEqual(test_data, new_data)

    def test_transform_field_serialization(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        site_slug_field = IPFabricTransformField.objects.get(
            source_field="siteName", target_field="slug"
        )
        data = serializers.serialize("json", [site_slug_field])
        data = json.loads(data)[0]
        test_data = {
            "model": "ipfabric_netbox.ipfabrictransformfield",
            "pk": site_slug_field.pk,
            "fields": {
                "source_field": "siteName",
                "target_field": "slug",
                "template": "{{ object.siteName | slugify }}",
                "transform_map": site_transform_map.pk,
            },
        }
        new_data = copy.deepcopy(data)
        new_data.pop("fields")
        new_data["fields"] = {k: data["fields"][k] for k in test_data["fields"]}
        self.assertDictEqual(test_data, new_data)

    def test_update_or_create_instance_site(self):
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        data = {
            "siteName": "Site 1",
            "devicesCount": 1,
            "usersCount": 2,
            "stpDCount": 0,
            "switchesCount": 0,
            "vlanCount": 1,
            "rDCount": 0,
            "routersCount": 0,
            "networksCount": 6,
        }
        context = site_transform_map.get_context(data)
        object = site_transform_map.update_or_create_instance(context)
        self.assertEqual(object.name, "Site 1")
        self.assertEqual(object.slug, "site-1")

    def test_update_or_create_instance_device(self):
        device_object = Device.objects.first()

        self.assertEqual(device_object.name, "L21PE152")
        self.assertEqual(device_object.serial, "a15ff98")
        self.assertEqual(device_object.platform, self.platform)
        self.assertEqual(device_object.role, self.role)
        self.assertEqual(device_object.device_type, self.dt_obj)
        self.assertEqual(device_object.device_type.manufacturer, self.mf_obj)
        self.assertEqual(device_object.site, self.site)
        self.assertEqual(device_object.status, "active")

    def test_alter_transform_field_template(self):
        sync = IPFabricSync.objects.get(name="ingest")

        runner = IPFabricSyncRunner(
            settings={
                "dcim.site": True,
                "groups": [],
                "dcim.device": True,
                "dcim.platform": True,
                "dcim.interface": False,
                "dcim.devicerole": True,
                "dcim.devicetype": True,
                "dcim.manufacturer": True,
                "dcim.virtualchassis": True,
                "snapshot_id": "12dd8c61-129c-431a-b98b-4c9211571f89",
            },
            sync=sync,
        )
        # Need to monkeypatch since we are not in active Branch (different schema)
        # Using default schema "default" here since we are in "test_netbox" DB
        runner.get_db_connection_name = lambda: DEFAULT_DB_ALIAS

        device_data = {
            "id": "961251111",
            "configReg": "0x0",
            "devType": "router",
            "family": "ios",
            "hostname": "L21PE152",
            "hostnameOriginal": None,
            "hostnameProcessed": None,
            "domain": None,
            "fqdn": None,
            "icon": None,
            "image": "unix:/opt/unetlab/addons/iol/bin/i86bi-linux-l3-adventerprisek9-15.2",
            "objectId": None,
            "taskKey": "fb67e3b4-5e48-4e52-b000-56cb187f2852",
            "loginIp": "10.21.254.152",
            "loginType": "telnet",
            "loginPort": None,
            "mac": None,
            "memoryTotalBytes": 396008048,
            "memoryUsedBytes": 72264172,
            "memoryUtilization": 18.25,
            "model": "",
            "platform": "i86bi_linux",
            "processor": "Intel-x86",
            "rd": "3",
            "reload": "reload at 0",
            "siteName": "MPLS",
            "sn": "a15ff98",
            "snHw": "a15ff98",
            "stpDomain": None,
            "uptime": 7254180,
            "vendor": "cisco",
            "version": "15.2(4)M1",
            "virtual_chassis": None,
            "slug": None,
        }

        device_endpoint = IPFabricEndpoint.objects.get(endpoint="/inventory/devices")
        transform_field = IPFabricTransformField.objects.get(
            source_field="hostname",
            target_field="name",
            transform_map=IPFabricTransformMap.objects.get(
                source_endpoint=device_endpoint,
                target_model=ContentType.objects.get(app_label="dcim", model="device"),
            ),
        )
        transform_field.template = "{{ object.hostname }} - test"
        transform_field.save()
        device_object = runner.sync_model(
            record=runner.create_new_data_record(
                app="dcim", model="device", data=device_data
            ),
        )
        self.assertEqual(device_object.name, "L21PE152 - test")

    # ------------------------------------------------------------------
    # get_model_hierarchy tests
    # ------------------------------------------------------------------

    def test_group_override_does_not_raise_false_circular_dependency(self):
        """
        Default maps (no group):
            A: parts        -> inventory_item   (no parents)
            B: interfaces   -> interface        (parents=[A])
            C: interfaces   -> macaddress       (parents=[A])
            D: managed_ip   -> ipaddress        (parents=[B, C])

          Group G overrides B, C, D with clones B', C', D'.
          The clones are created with parents pointing to the *original* default
          A/B/C IDs (as a UI clone would do).

        The hierarchy should be resolved correctly with no error.
        """
        A = self._default_map(
            "Default A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Default B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        C = self._default_map("Default C (ifaces->mac)", self.ep_ifaces, self.ct_mac)
        D = self._default_map(
            "Default D (mgdip->ipaddr)", self.ep_managed_ip, self.ct_ipaddress
        )

        # Wire default hierarchy: B->A, C->A, D->B+C
        B.parents.add(A)
        C.parents.add(A)
        D.parents.add(B, C)

        group = IPFabricTransformMapGroup.objects.create(name="Override Group")

        # Clone B, C, D into the group; parents intentionally still point to *default* IDs
        B_prime = self._group_map(
            "Group B' (ifaces->iface)", self.ep_ifaces, self.ct_interface, group
        )
        C_prime = self._group_map(
            "Group C' (ifaces->mac)", self.ep_ifaces, self.ct_mac, group
        )
        D_prime = self._group_map(
            "Group D' (mgdip->ipaddr)", self.ep_managed_ip, self.ct_ipaddress, group
        )

        # Clones point to the ORIGINAL default IDs (this is what a manual clone produces)
        B_prime.parents.add(A)  # A is not overridden — still valid
        C_prime.parents.add(A)
        D_prime.parents.add(B, C)  # B and C are the *default* IDs, not B'/C'

        # Should NOT raise
        result = IPFabricSync.get_model_hierarchy(group_ids=[group.id])

        active_ids = {tm.id for tm in result}

        # Active set must contain the group overrides, not the originals
        self.assertIn(B_prime.id, active_ids)
        self.assertIn(C_prime.id, active_ids)
        self.assertIn(D_prime.id, active_ids)
        self.assertNotIn(B.id, active_ids)
        self.assertNotIn(C.id, active_ids)
        self.assertNotIn(D.id, active_ids)

        # Dependency ordering: A before B', C'; B' and C' before D'
        idx = {tm.id: i for i, tm in enumerate(result)}
        self.assertLess(idx[A.id], idx[B_prime.id])
        self.assertLess(idx[A.id], idx[C_prime.id])
        self.assertLess(idx[B_prime.id], idx[D_prime.id])
        self.assertLess(idx[C_prime.id], idx[D_prime.id])

    def test_no_groups_default_hierarchy_unchanged(self):
        """Without any groups the existing default hierarchy must work as before."""
        A = self._default_map(
            "NoGrp A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "NoGrp B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        B.parents.add(A)

        result = IPFabricSync.get_model_hierarchy(group_ids=[])
        ids = [tm.id for tm in result]

        self.assertIn(A.id, ids)
        self.assertIn(B.id, ids)
        self.assertLess(ids.index(A.id), ids.index(B.id))

    def test_group_fully_overrides_chain_preserves_order(self):
        """
        When a group overrides every map in a chain the order must still be correct
        and no error raised.
        """
        A = self._default_map(
            "Chain A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Chain B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        B.parents.add(A)

        group = IPFabricTransformMapGroup.objects.create(name="Full Override Group")

        A_prime = self._group_map(
            "Chain A' (parts->invitem)", self.ep_parts, self.ct_inventory_item, group
        )
        B_prime = self._group_map(
            "Chain B' (ifaces->iface)", self.ep_ifaces, self.ct_interface, group
        )

        # B' parent is the original A (not A') — the typical cloning scenario
        B_prime.parents.add(A)

        result = IPFabricSync.get_model_hierarchy(group_ids=[group.id])
        idx = {tm.id: i for i, tm in enumerate(result)}

        # Both overrides must be present; originals must not
        self.assertIn(A_prime.id, idx)
        self.assertIn(B_prime.id, idx)
        self.assertNotIn(A.id, idx)
        self.assertNotIn(B.id, idx)
        # Order preserved: A' before B'
        self.assertLess(idx[A_prime.id], idx[B_prime.id])

    def test_true_circular_dependency_still_raises(self):
        """
        A genuine cycle in default maps must still raise ValidationError.
        The cycle is injected directly via the M2M through table to bypass clean().
        """
        A = self._default_map(
            "Cycle A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Cycle B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )

        # A -> B -> A  (cycle)
        IPFabricTransformMap.parents.through.objects.create(
            from_ipfabrictransformmap=A,
            to_ipfabrictransformmap=B,
        )
        IPFabricTransformMap.parents.through.objects.create(
            from_ipfabrictransformmap=B,
            to_ipfabrictransformmap=A,
        )

        with self.assertRaises(ValidationError) as ctx:
            IPFabricSync.get_model_hierarchy(group_ids=[])

        self.assertIn("Circular dependency", str(ctx.exception))

    def test_m2m_signal_raises_on_cycle(self):
        """
        validate_circular_dependency_on_m2m_change fires on 'pre_add' and must
        raise ValidationError when parents.add() would close a cycle.
        This is a distinct code path from clean() — it catches programmatic /
        API mutations that bypass form validation entirely.

        Chain before the bad add:  A -> B -> C  (valid, no cycle)
        After A.parents.add(C):    A -> B -> C -> A  (cycle)
        """
        A = self._default_map(
            "Signal Cycle A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Signal Cycle B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        C = self._default_map(
            "Signal Cycle C (ifaces->mac)", self.ep_ifaces, self.ct_mac
        )

        # Build a valid three-node chain A -> B -> C
        B.parents.add(A)
        C.parents.add(B)

        # Adding C as a parent of A would close the cycle A -> B -> C -> A
        with self.assertRaises(ValidationError) as ctx:
            A.parents.add(C)

        self.assertIn("circular dependency detected", str(ctx.exception).lower())

    def test_m2m_signal_allows_valid_parent_add(self):
        """
        validate_circular_dependency_on_m2m_change must NOT raise when the
        new parent relationship is acyclic.
        """
        A = self._default_map(
            "Signal Valid A (parts->invitem)", self.ep_parts, self.ct_inventory_item
        )
        B = self._default_map(
            "Signal Valid B (ifaces->iface)", self.ep_ifaces, self.ct_interface
        )
        C = self._default_map(
            "Signal Valid C (ifaces->mac)", self.ep_ifaces, self.ct_mac
        )

        # B depends on A — valid
        B.parents.add(A)

        # C depends on B — still valid, no cycle introduced
        C.parents.add(B)  # must not raise

        self.assertIn(B, list(C.parents.all()))

    def test_template_syntax_error_caught_in_relationship_field_clean(self):
        """IPFabricRelationshipField.clean() should catch template syntax errors."""
        # Use existing Device Transform Map
        device_endpoint = IPFabricEndpoint.objects.get(endpoint="/inventory/devices")
        device_transform_map = IPFabricTransformMap.objects.get(
            source_endpoint=device_endpoint,
            target_model=ContentType.objects.get(app_label="dcim", model="device"),
        )

        # Invalid Jinja2 syntax (unclosed tag)
        rel_field = IPFabricRelationshipField(
            transform_map=device_transform_map,
            source_model=ContentType.objects.get(app_label="dcim", model="site"),
            target_field="site",
            template="{{ object.siteName",  # Missing closing braces
        )

        with self.assertRaises(ValidationError) as ctx:
            rel_field.clean()

        self.assertIn("template", ctx.exception.message_dict)
        self.assertIn("Template syntax error", str(ctx.exception))

    def test_template_syntax_error_caught_in_transform_field_clean(self):
        """IPFabricTransformField.clean() should catch template syntax errors."""
        # Use existing Site Transform Map
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")

        # Invalid Jinja2 syntax (undefined filter)
        field = IPFabricTransformField(
            transform_map=site_transform_map,
            source_field="siteName",
            target_field="name",
            template="{{ object.siteName | nonexistent_filter }}",
        )

        with self.assertRaises(ValidationError) as ctx:
            field.clean()

        self.assertIn("template", ctx.exception.message_dict)
        self.assertIn("Template syntax error", str(ctx.exception))

    def test_valid_template_with_orm_query_passes_clean(self):
        """Valid templates that include ORM queries should pass clean() even if queries fail."""
        # Use existing Site Transform Map
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")

        # Valid template with ORM query (will fail at runtime with empty context, but syntax is valid)
        field = IPFabricTransformField(
            transform_map=site_transform_map,
            source_field="siteName",
            target_field="slug",
            template="{{ dcim.Site.objects.get(name=object.siteName).pk }}",
        )

        # Should not raise ValidationError - ORM exceptions are allowed during validation
        try:
            field.clean()
        except ValidationError:
            self.fail(
                "clean() should not raise ValidationError for valid template syntax"
            )

    def test_template_with_source_field_reference_passes_clean(self):
        """Templates referencing object.source_field should pass clean() without UndefinedError."""
        # Use existing Site Transform Map
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")

        # Template that references object.siteName - should not raise UndefinedError
        field = IPFabricTransformField(
            transform_map=site_transform_map,
            source_field="siteName",
            target_field="slug",
            template="{{ object.siteName | slugify }}",
        )

        # Should not raise ValidationError or UndefinedError
        try:
            field.clean()
        except ValidationError as e:
            self.fail(
                f"clean() should not raise ValidationError for valid template: {e}"
            )

    # ------------------------------------------------------------------
    # Custom field (cf_ipfabric_source / cf_ipfabric_ingestion) tests.
    # Both CFs and the transform fields that write them are created by
    # migrations — no setup needed here.
    # ------------------------------------------------------------------

    def _site_data_with_ingestion(self, ingestion):
        """Minimal site source-data dict with the ingestion object injected."""
        return {
            "siteName": "MPLS",
            "ingestion": ingestion,
        }

    def test_cf_source_and_ingestion_written_on_create(self):
        """cf_ipfabric_source and cf_ipfabric_ingestion are populated when a Site is first synced."""
        source = IPFabricSource.objects.get(name="test")
        ingestion = IPFabricIngestion.objects.filter(sync__name="ingest").first()

        self.assertEqual(self.site.custom_field_data["ipfabric_source"], source.pk)
        self.assertEqual(
            self.site.custom_field_data["ipfabric_ingestion"], ingestion.pk
        )

    def test_cf_source_and_ingestion_updated_on_resync(self):
        """cf_ipfabric_source and cf_ipfabric_ingestion are updated when the site is re-synced."""
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        new_ingestion = IPFabricIngestion.objects.create(
            sync=IPFabricSync.objects.get(name="ingest")
        )
        original_ingestion_pk = self.site.custom_field_data["ipfabric_ingestion"]

        context = site_transform_map.get_context(
            self._site_data_with_ingestion(new_ingestion)
        )
        site = site_transform_map.update_or_create_instance(context)

        self.assertNotEqual(new_ingestion.pk, original_ingestion_pk)
        self.assertEqual(
            site.custom_field_data["ipfabric_source"],
            new_ingestion.sync.snapshot_data.source.pk,
        )
        self.assertEqual(site.custom_field_data["ipfabric_ingestion"], new_ingestion.pk)

    def test_unrelated_cf_data_preserved_during_sync(self):
        """CF keys not managed by IPFabric are not overwritten when the site is re-synced."""
        external_cf = CustomField.objects.create(
            name="test_external_cf",
            type="text",
            label="Test External CF",
        )
        site_ct = ContentType.objects.get(app_label="dcim", model="site")
        external_cf.object_types.set([site_ct])

        self.site.custom_field_data["test_external_cf"] = "keep-me"
        self.site.save()

        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")
        new_ingestion = IPFabricIngestion.objects.create(
            sync=IPFabricSync.objects.get(name="ingest")
        )

        context = site_transform_map.get_context(
            self._site_data_with_ingestion(new_ingestion)
        )
        site = site_transform_map.update_or_create_instance(context)

        self.assertEqual(site.custom_field_data["ipfabric_ingestion"], new_ingestion.pk)
        self.assertEqual(site.custom_field_data.get("test_external_cf"), "keep-me")

    def test_cf_coalesce_key_translated_to_orm_lookup(self):
        """A cf_* key in the context is translated to custom_field_data__* for the ORM lookup."""
        source = IPFabricSource.objects.get(name="test")
        ingestion = IPFabricIngestion.objects.filter(sync__name="ingest").first()
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")

        # Craft a context as if cf_ipfabric_source were a coalesce (lookup) field.
        # update_or_create_instance must translate it to
        # custom_field_data__ipfabric_source=source.pk so the ORM query finds self.site.
        context = {
            "cf_ipfabric_source": source.pk,
            "defaults": {
                "name": "MPLS",
                "cf_ipfabric_ingestion": ingestion.pk,
            },
        }
        site = site_transform_map.update_or_create_instance(context)
        self.assertEqual(site.name, "MPLS")

    def test_cf_coalesce_isnull_key_translated_to_orm_lookup(self):
        """A cf_*__isnull key in the context is translated to custom_field_data__*__isnull."""
        ingestion = IPFabricIngestion.objects.filter(sync__name="ingest").first()
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")

        # Remove the key entirely — PostgreSQL jsonb __isnull=True matches absent keys
        # (→ 'key' returns SQL NULL), not keys whose value is JSON null.
        self.site.custom_field_data.pop("ipfabric_source")
        self.site.save()

        # Craft a context as if a coalesce CF field rendered to None (producing __isnull=True).
        # slug disambiguates from any other site that might also have ipfabric_source=None.
        context = {
            "slug": "mpls",
            "cf_ipfabric_source__isnull": True,
            "defaults": {
                "name": "MPLS",
                "cf_ipfabric_ingestion": ingestion.pk,
            },
        }
        site = site_transform_map.update_or_create_instance(context)
        self.assertEqual(site.slug, "mpls")

    def test_cf_coalesce_creates_new_object_with_cf_value(self):
        """When a cf_* key is used as a coalesce field and the object does not exist,
        it is created and the custom field value from the coalesce is set on the new instance.
        """
        source = IPFabricSource.objects.get(name="test")
        ingestion = IPFabricIngestion.objects.filter(sync__name="ingest").first()
        site_transform_map = IPFabricTransformMap.objects.get(name="Site Transform Map")

        # Use a slug that does not exist yet so the DoesNotExist branch is triggered.
        context = {
            "slug": "brand-new-cf-site",
            "cf_ipfabric_source": source.pk,
            "defaults": {
                "name": "Brand New CF Site",
                "cf_ipfabric_ingestion": ingestion.pk,
            },
        }
        site = site_transform_map.update_or_create_instance(context)

        self.assertEqual(site.name, "Brand New CF Site")
        # The CF coalesce value must be persisted on the created object.
        self.assertEqual(site.custom_field_data["ipfabric_source"], source.pk)
        # Defaults CF values must also be set.
        self.assertEqual(site.custom_field_data["ipfabric_ingestion"], ingestion.pk)


class IPFabricFilterCombinationTestCase(TestCase):
    """Test cases for recursive filter merging with nested and/or structures."""

    @classmethod
    def setUpTestData(cls):
        # Create source
        cls.source = IPFabricSource.objects.create(
            name="Test Source",
            url="https://test.ipfabric.io",
            status="active",
            parameters={"auth": "test_token", "verify": False},
        )
        # Create snapshot
        cls.snapshot = IPFabricSnapshot.objects.create(
            name="Test Snapshot",
            source=cls.source,
            snapshot_id="test-snapshot-123",
            status=IPFabricSnapshotStatusModelChoices.STATUS_LOADED,
            data={"sites": ["Site1", "Site2"]},
        )
        # Create sync
        cls.sync = IPFabricSync.objects.create(
            name="Test Sync",
            snapshot_data=cls.snapshot,
            status="new",
        )
        # Get or create endpoint
        cls.endpoint, _ = IPFabricEndpoint.objects.get_or_create(
            endpoint="/inventory/devices",
            defaults={"name": "Devices Endpoint"},
        )

    def test_combine_filters_merges_same_nested_or_structure(self):
        """Test that filters with same nested 'or' structure merge correctly."""
        # Create two AND-type filters
        filter1 = IPFabricFilter.objects.create(
            name="Filter with router1",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        filter2 = IPFabricFilter.objects.create(
            name="Filter with router2",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter2.endpoints.add(self.endpoint)
        filter2.syncs.add(self.sync)

        # Create expressions with same nested structure
        expr1 = IPFabricFilterExpression.objects.create(
            name="Expression router1",
            expression=[{"or": [{"hostname": ["like", "router1"]}]}],
        )
        expr1.filters.add(filter1)

        expr2 = IPFabricFilterExpression.objects.create(
            name="Expression router2",
            expression=[{"or": [{"hostname": ["like", "router2"]}]}],
        )
        expr2.filters.add(filter2)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have merged into single nested structure
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])
        self.assertEqual(len(result["and"][0]["or"]), 2)

        # Verify both hostnames are present
        hostnames = [item["hostname"][1] for item in result["and"][0]["or"]]
        self.assertIn("router1", hostnames)
        self.assertIn("router2", hostnames)

    def test_combine_filters_deeply_nested_structures(self):
        """Test that deeply nested structures (3+ levels) merge correctly."""
        # Create two AND-type filters with 3-level nesting
        filter1 = IPFabricFilter.objects.create(
            name="Deep filter cisco",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        filter2 = IPFabricFilter.objects.create(
            name="Deep filter juniper",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter2.endpoints.add(self.endpoint)
        filter2.syncs.add(self.sync)

        # Create deeply nested expressions
        expr1 = IPFabricFilterExpression.objects.create(
            name="Deep expression cisco",
            expression=[{"or": [{"and": [{"or": [{"vendor": ["eq", "cisco"]}]}]}]}],
        )
        expr1.filters.add(filter1)

        expr2 = IPFabricFilterExpression.objects.create(
            name="Deep expression juniper",
            expression=[{"or": [{"and": [{"or": [{"vendor": ["eq", "juniper"]}]}]}]}],
        )
        expr2.filters.add(filter2)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have merged at all levels
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])
        self.assertEqual(len(result["and"][0]["or"]), 1)
        self.assertIn("and", result["and"][0]["or"][0])
        self.assertEqual(len(result["and"][0]["or"][0]["and"]), 1)
        self.assertIn("or", result["and"][0]["or"][0]["and"][0])

        # Verify both vendors are in innermost or array
        innermost_or = result["and"][0]["or"][0]["and"][0]["or"]
        self.assertEqual(len(innermost_or), 2)
        vendors = [item["vendor"][1] for item in innermost_or]
        self.assertIn("cisco", vendors)
        self.assertIn("juniper", vendors)

    def test_combine_filters_different_structures_no_merge(self):
        """Test that filters with different structures don't merge."""
        # Create two AND-type filters with different nested structures
        filter1 = IPFabricFilter.objects.create(
            name="Filter with or structure",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        filter2 = IPFabricFilter.objects.create(
            name="Filter with and structure",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter2.endpoints.add(self.endpoint)
        filter2.syncs.add(self.sync)

        # Create expressions with different structures
        expr1 = IPFabricFilterExpression.objects.create(
            name="Expression with or",
            expression=[{"or": [{"hostname": ["like", "router1"]}]}],
        )
        expr1.filters.add(filter1)

        expr2 = IPFabricFilterExpression.objects.create(
            name="Expression with and",
            expression=[{"and": [{"hostname": ["like", "router2"]}]}],
        )
        expr2.filters.add(filter2)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have two separate structures (not merged)
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 2)

        # One should have 'or', the other should have 'and'
        structures = [list(item.keys()) for item in result["and"]]
        self.assertIn(["or"], structures)
        self.assertIn(["and"], structures)

    def test_combine_filters_multiple_filters_same_structure(self):
        """Test that 3+ filters with same structure all merge correctly."""
        # Create three AND-type filters with same nested structure
        for i in range(1, 4):
            filter_obj = IPFabricFilter.objects.create(
                name=f"Filter with router{i}",
                filter_type=IPFabricFilterTypeChoices.AND,
            )
            filter_obj.endpoints.add(self.endpoint)
            filter_obj.syncs.add(self.sync)

            expr = IPFabricFilterExpression.objects.create(
                name=f"Expression router{i}",
                expression=[{"or": [{"hostname": ["like", f"router{i}"]}]}],
            )
            expr.filters.add(filter_obj)

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have merged all three into single nested structure
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])
        self.assertEqual(len(result["and"][0]["or"]), 3)

        # Verify all three hostnames are present
        hostnames = [item["hostname"][1] for item in result["and"][0]["or"]]
        self.assertIn("router1", hostnames)
        self.assertIn("router2", hostnames)
        self.assertIn("router3", hostnames)

    def test_combine_filters_sites_merge_with_nested_or(self):
        """Test that sync sites parameter merges correctly with existing nested or expressions.

        This test validates the fix for a bug where site filters from sync.parameters
        were not being merged when another nested 'or' expression already existed in
        an 'and' filter. The correct behavior is to merge the site conditions into the
        existing 'or' array within the 'and' structure.
        """
        # Create an AND-type filter with nested or structure
        filter1 = IPFabricFilter.objects.create(
            name="Filter with nested or",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        # Create expression with nested or structure
        expr1 = IPFabricFilterExpression.objects.create(
            name="Expression with test field",
            expression=[{"or": [{"test": ["eq", 123]}]}],
        )
        expr1.filters.add(filter1)

        # Configure sync with sites parameter (list of site names)
        self.sync.parameters = {"sites": ["L89"]}
        self.sync.save()

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have 'and' at top level with merged structure
        self.assertIn("and", result)
        self.assertEqual(len(result["and"]), 1)

        # Should have one 'or' structure containing both test and siteName
        self.assertIn("or", result["and"][0])
        or_conditions = result["and"][0]["or"]
        self.assertEqual(len(or_conditions), 2)

        # Verify test field is present
        test_found = False
        for condition in or_conditions:
            if "test" in condition:
                self.assertEqual(condition["test"], ["eq", 123])
                test_found = True

        self.assertTrue(test_found, "Test field not found in combined filter")

        # Verify siteName is present
        sitename_found = False
        for condition in or_conditions:
            if "siteName" in condition:
                self.assertEqual(condition["siteName"], ["eq", "L89"])
                sitename_found = True

        self.assertTrue(sitename_found, "siteName filter not found in combined filter")

    def test_combine_filters_sites_empty_list(self):
        """Test that empty sites list doesn't add filters or cause errors."""
        # Create a basic filter
        filter1 = IPFabricFilter.objects.create(
            name="Basic filter",
            filter_type=IPFabricFilterTypeChoices.AND,
        )
        filter1.endpoints.add(self.endpoint)
        filter1.syncs.add(self.sync)

        expr1 = IPFabricFilterExpression.objects.create(
            name="Basic expression",
            expression=[{"hostname": ["like", "router"]}],
        )
        expr1.filters.add(filter1)

        # Configure sync with empty sites list
        self.sync.parameters = {"sites": []}
        self.sync.save()

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should only have the original filter, no sites filter
        self.assertIn("and", result)
        # Should be a list with just the hostname condition, not nested
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("hostname", result["and"][0])

    def test_combine_filters_sites_with_multiple_existing_or_structures(self):
        """Test that sites merge correctly when multiple 'or' structures already exist.

        When multiple filters with the same nested 'or' structure exist, they merge together,
        and then the sites parameter merges into that same structure.
        """
        # Create three AND-type filters, each with nested or structure
        for i in range(1, 4):
            filter_obj = IPFabricFilter.objects.create(
                name=f"Filter with or {i}",
                filter_type=IPFabricFilterTypeChoices.AND,
            )
            filter_obj.endpoints.add(self.endpoint)
            filter_obj.syncs.add(self.sync)

            expr = IPFabricFilterExpression.objects.create(
                name=f"Expression {i}",
                expression=[{"or": [{"hostname": ["like", f"router{i}"]}]}],
            )
            expr.filters.add(filter_obj)

        # Configure sync with sites parameter
        self.sync.parameters = {"sites": ["Site1", "Site2"]}
        self.sync.save()

        # Combine filters
        result = self.endpoint.combine_filters(sync=self.sync)

        # Should have 'and' at top level
        self.assertIn("and", result)

        # All filters should merge into one 'or' structure since they have the same nested format
        self.assertEqual(len(result["and"]), 1)
        self.assertIn("or", result["and"][0])

        # The 'or' array should contain 3 hostnames + 2 sites = 5 conditions total
        or_conditions = result["and"][0]["or"]
        self.assertEqual(len(or_conditions), 5)

        # Verify all three hostnames are present
        hostnames = [
            item["hostname"][1] for item in or_conditions if "hostname" in item
        ]
        self.assertEqual(len(hostnames), 3)
        self.assertIn("router1", hostnames)
        self.assertIn("router2", hostnames)
        self.assertIn("router3", hostnames)

        # Verify both sites are present
        sites = [item["siteName"][1] for item in or_conditions if "siteName" in item]
        self.assertEqual(len(sites), 2)
        self.assertIn("Site1", sites)
        self.assertIn("Site2", sites)


class SyncLoggingTestCase(TestCase):
    """Tests for the SyncLogging utility class."""

    def setUp(self):
        self.logger = SyncLogging(key_prefix="test_sync", job=999)

    def test_initial_log_data_structure(self):
        """log_data must contain a single statistics key (no phase split)."""
        self.assertIn("logs", self.logger.log_data)
        self.assertIn("statistics", self.logger.log_data)
        self.assertNotIn("sync_statistics", self.logger.log_data)
        self.assertNotIn("merge_statistics", self.logger.log_data)
        self.assertEqual(self.logger.log_data["statistics"], {})

    def test_init_statistics(self):
        """init_statistics creates entry under statistics with current=0."""
        self.logger.init_statistics("dcim.device", total=10)
        self.assertIn("dcim.device", self.logger.log_data["statistics"])
        self.assertEqual(
            self.logger.log_data["statistics"]["dcim.device"],
            {"current": 0, "total": 10},
        )

    def test_init_statistics_does_not_overwrite(self):
        """init_statistics is idempotent — a second call does not reset current."""
        self.logger.init_statistics("dcim.device", total=10)
        self.logger.increment_statistics("dcim.device")
        self.logger.init_statistics("dcim.device", total=99)  # should be a no-op
        self.assertEqual(
            self.logger.log_data["statistics"]["dcim.device"],
            {"current": 1, "total": 10},
        )

    def test_increment_statistics(self):
        """increment_statistics increments current counter."""
        self.logger.init_statistics("dcim.device", total=5)
        self.logger.increment_statistics("dcim.device")
        self.logger.increment_statistics("dcim.device")
        self.assertEqual(
            self.logger.log_data["statistics"]["dcim.device"]["current"], 2
        )

    def test_increment_statistics_auto_creates_entry(self):
        """increment_statistics creates the entry if not pre-initialised."""
        self.logger.increment_statistics("ipam.prefix")
        self.assertIn("ipam.prefix", self.logger.log_data["statistics"])
        self.assertEqual(
            self.logger.log_data["statistics"]["ipam.prefix"]["current"], 1
        )


class MergeBranchTestCase(TestCase):
    """Tests for the merge_branch() utility function."""

    def _make_ingestion(self, branch=None):
        source = IPFabricSource.objects.create(
            name="Merge Test Source",
            url="https://merge.test.local",
            parameters={"auth": "token", "verify": True},
        )
        snapshot = IPFabricSnapshot.objects.create(
            name="Merge Test Snapshot",
            source=source,
            snapshot_id="merge-snap-1",
            status="loaded",
            data={},
        )
        user = get_user_model().objects.create_user(username="merge-test-user")
        sync = IPFabricSync.objects.create(
            name="Merge Test Sync",
            snapshot_data=snapshot,
            parameters={},
            user=user,
        )
        ingestion = IPFabricIngestion.objects.create(sync=sync, branch=branch)
        sync.logger = SyncLogging(key_prefix="test_merge", job=sync.pk)
        return ingestion

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.BranchEvent.objects")
    @patch("ipfabric_netbox.utilities.merge.post_merge")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    def test_no_changes_aborts_early(
        self, mock_post_save, mock_post_merge, mock_branch_event, mock_branch_objects
    ):
        """merge_branch returns early without touching the branch when there are no changes."""
        branch = MagicMock()
        branch.ready = True
        branch.get_unmerged_changes.return_value.order_by.return_value.count.return_value = (
            0
        )
        ingestion = MagicMock()
        ingestion.branch = branch

        merge_branch(ingestion)

        mock_branch_objects.filter.assert_not_called()
        mock_post_merge.send.assert_not_called()
        mock_branch_event.create.assert_not_called()

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.BranchEvent.objects")
    @patch("ipfabric_netbox.utilities.merge.post_merge")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    @patch("ipfabric_netbox.utilities.merge.connection")
    @patch("ipfabric_netbox.utilities.merge.get_merge_strategy")
    def test_successful_change_applied(
        self,
        mock_get_strategy,
        mock_connection,
        mock_post_save,
        mock_post_merge,
        mock_branch_event,
        mock_branch_objects,
    ):
        """A change that applies without error is committed and counted in merge_statistics."""
        mock_strategy = MagicMock()
        mock_get_strategy.return_value.return_value = mock_strategy
        mock_connection.savepoint.return_value = "sp1"

        change = MagicMock()
        change.pk = 42
        change.action = "create"
        change.changed_object_id = 1
        ct = MagicMock()
        ct.natural_key.return_value = ("dcim", "device")
        ct.model_class.return_value = MagicMock()
        change.changed_object_type = ct

        branch = MagicMock()
        branch.ready = True
        branch.schema_name = "branch_abc"
        branch.merge_strategy = None
        qs = MagicMock()
        qs.count.return_value = 1
        qs.values_list.return_value = [("dcim", "device")]
        qs.__iter__ = MagicMock(return_value=iter([change]))
        branch.get_unmerged_changes.return_value.order_by.return_value = qs

        ingestion = MagicMock()
        ingestion.branch = branch
        sync_logger = SyncLogging(key_prefix="test", job=1)
        ingestion.sync.user = MagicMock()

        merge_branch(ingestion, sync_logger=sync_logger)

        mock_connection.savepoint_commit.assert_called_once_with("sp1")
        mock_connection.savepoint_rollback.assert_not_called()
        self.assertEqual(
            sync_logger.log_data["statistics"]["dcim.device"]["current"], 1
        )
        mock_branch_event.create.assert_called_once()
        mock_post_merge.send.assert_called_once()

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.BranchEvent.objects")
    @patch("ipfabric_netbox.utilities.merge.post_merge")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    @patch("ipfabric_netbox.utilities.merge.connection")
    @patch("ipfabric_netbox.utilities.merge.get_merge_strategy")
    @patch("ipfabric_netbox.utilities.merge.IPFabricIngestionIssue")
    def test_failed_change_creates_issue_and_continues(
        self,
        mock_issue_class,
        mock_get_strategy,
        mock_connection,
        mock_post_save,
        mock_post_merge,
        mock_branch_event,
        mock_branch_objects,
    ):
        """A change that raises is rolled back, recorded as IPFabricIngestionIssue, and
        the loop continues to apply the next change."""
        mock_strategy = MagicMock()
        mock_get_strategy.return_value.return_value = mock_strategy
        mock_connection.savepoint.side_effect = ["sp1", "sp2"]

        def make_change(pk, action, should_fail):
            change = MagicMock()
            change.pk = pk
            change.action = action
            change.changed_object_id = pk
            ct = MagicMock()
            ct.natural_key.return_value = ("dcim", "device")
            ct.model_class.return_value = MagicMock()
            change.changed_object_type = ct
            if should_fail:
                change.apply.side_effect = ValueError("boom")
            return change

        bad_change = make_change(1, "create", should_fail=True)
        good_change = make_change(2, "update", should_fail=False)

        branch = MagicMock()
        branch.ready = True
        branch.schema_name = "branch_xyz"
        branch.merge_strategy = None
        qs = MagicMock()
        qs.count.return_value = 2
        qs.values_list.return_value = [("dcim", "device"), ("dcim", "device")]
        qs.__iter__ = MagicMock(return_value=iter([bad_change, good_change]))
        branch.get_unmerged_changes.return_value.order_by.return_value = qs

        ingestion = MagicMock()
        ingestion.branch = branch
        sync_logger = SyncLogging(key_prefix="fail_test", job=1)

        with self.assertLogs("ipfabric_netbox.merge", level="ERROR") as log:
            merge_branch(ingestion, sync_logger=sync_logger)

        self.assertTrue(any("boom" in msg for msg in log.output))

        # Bad change: savepoint rolled back
        mock_connection.savepoint_rollback.assert_called_once_with("sp1")
        # Good change: savepoint committed
        mock_connection.savepoint_commit.assert_called_once_with("sp2")

        # One issue created for the bad change — assert on the create() call kwargs
        mock_issue_class.objects.create.assert_called_once()
        call_kwargs = mock_issue_class.objects.create.call_args.kwargs
        self.assertEqual(call_kwargs["phase"], IPFabricIngestionPhaseChoices.MERGE)
        self.assertEqual(call_kwargs["model"], "dcim.device")
        self.assertIn("boom", call_kwargs["message"])
        self.assertEqual(call_kwargs["exception"], "ValueError")

        # Merge still completed (BranchEvent + post_merge fired)
        mock_branch_event.create.assert_called_once()
        mock_post_merge.send.assert_called_once()

        # Progress: 1 success out of 2 total
        self.assertEqual(
            sync_logger.log_data["statistics"]["dcim.device"]["current"], 1
        )

    @patch("ipfabric_netbox.utilities.merge.Branch.objects")
    @patch("ipfabric_netbox.utilities.merge.post_save")
    def test_signal_disconnected_on_exception(
        self, mock_post_save, mock_branch_objects
    ):
        """record_applied_change signal is disconnected even when an unexpected
        exception escapes the loop (e.g. from branch status update)."""
        branch = MagicMock()
        branch.ready = True
        branch.schema_name = "branch_err"
        qs = MagicMock()
        qs.count.return_value = 1
        qs.values_list.return_value = [("dcim", "site")]
        # Make iteration itself raise to simulate an unexpected error
        qs.__iter__ = MagicMock(side_effect=RuntimeError("unexpected"))
        branch.get_unmerged_changes.return_value.order_by.return_value = qs

        ingestion = MagicMock()
        ingestion.branch = branch

        with self.assertRaises(RuntimeError):
            merge_branch(ingestion)

        # disconnect must have been called despite the exception
        mock_post_save.disconnect.assert_called_once()
