import contextlib
import contextvars
import json
import logging
import math
import threading
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import wait as futures_wait
from copy import deepcopy
from functools import cache
from importlib import metadata
from typing import Callable
from typing import TYPE_CHECKING
from typing import TypeVar

from core.exceptions import SyncError
from core.signals import clear_events
from dcim.models import Device
from dcim.models import Site
from dcim.models import VirtualChassis
from dcim.signals import assign_virtualchassis_master
from django.db import close_old_connections
from django.db import connections
from rq.timeouts import JobTimeoutException

try:
    # Got added in NetBox 4.4.9
    from dcim.signals import sync_cached_scope_fields
except ImportError:
    sync_cached_scope_fields = None
from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Model
from django.db.models import signals
from django.utils.text import slugify
from django_tables2 import Column
from extras.events import flush_events
from ipfabric import IPFClient
from jinja2 import Template
from jinja2.sandbox import SandboxedEnvironment
from netbox.config import get_config
from netbox.context import events_queue
from netutils.utils import jinja2_convenience_function

from ..choices import IPFabricEndpointChoices
from ..choices import IPFabricSourceTypeChoices
from ..choices import IPFabricSyncStatusChoices
from ..exceptions import IPAddressDuplicateError
from ..exceptions import IPAddressPrimaryAssignmentError
from ..exceptions import IPAddressPrimaryRemovalError
from ..exceptions import RequiredDependencyFailedSkip
from ..exceptions import SearchError
from ..exceptions import SyncDataError


if TYPE_CHECKING:
    from ..models import IPFabricIngestion
    from ..models import IPFabricIngestionIssue
    from ipam.models import IPAddress
    from ..models import IPFabricTransformMap
    from ..models import IPFabricSync

logger = logging.getLogger("ipfabric_netbox.utilities.ipf_utils")

ModelTypeVar = TypeVar("ModelTypeVar", bound=Model)


def slugify_text(value):
    return slugify(value)


device_serial_max_length = Device._meta.get_field("serial").max_length


def serial(data: dict) -> str:
    sn_length = len(data.get("sn"))
    serial_number = data.get("sn") if sn_length < device_serial_max_length else ""
    if not serial_number:
        serial_number = data.get("id")
    return serial_number


IPF_JINJA_FILTERS = {"slugify": slugify_text, "serial": serial}


class Jinja2TemplateCache:
    """
    Singleton-style cache for the shared Jinja2 environment and compiled templates.

    Building a SandboxedEnvironment and registering all filters is expensive — it
    was previously done on *every* render_jinja2() call (once per record, per field).
    The environment is stateless after construction, so it is safe to share across
    threads.  Compiled Template objects are also thread-safe and are cached by
    template_code string so each unique template string is compiled exactly once.

    All state lives at class level so there is exactly one cache for the whole
    process, regardless of how many times the module is imported — no instances
    are ever needed; call Jinja2TemplateCache.render() directly.
    """

    _env: "SandboxedEnvironment | None" = None
    _env_lock = threading.Lock()
    _template_cache: dict[str, Template] = {}
    _template_cache_lock = threading.Lock()

    @classmethod
    def _get_env(cls) -> SandboxedEnvironment:
        """Return the shared, lazily-initialised Jinja2 environment."""
        if cls._env is None:
            with cls._env_lock:
                # Double-checked locking: another thread may have initialised it
                # while we were waiting for the lock.
                if cls._env is None:
                    env = SandboxedEnvironment()
                    env.filters.update(get_config().JINJA2_FILTERS)
                    env.filters.update(IPF_JINJA_FILTERS)
                    env.filters.update(jinja2_convenience_function())
                    cls._env = env
        return cls._env

    @classmethod
    def render(cls, template_code: str, context: dict) -> str:
        """Render a Jinja2 template with the provided context."""
        if template_code not in cls._template_cache:
            compiled = cls._get_env().from_string(source=template_code)
            with cls._template_cache_lock:
                # Another thread may have compiled it while we waited — that's fine,
                # we just overwrite with an equivalent object.
                cls._template_cache[template_code] = compiled
        return cls._template_cache[template_code].render(**context)


def render_jinja2(template_code: str, context: dict) -> str:
    """
    Render a Jinja2 template with the provided context. Return the rendered content.

    Delegates to Jinja2TemplateCache which keeps a single shared environment and
    a compiled-template cache across all calls (thread-safe).
    """
    return Jinja2TemplateCache.render(template_code, context)


class EventsClearer:
    """
    Handles clearing of events after defined number of objects are saved to reduce memory usage.
    ChangeLoggingMiddleware causes rest_framework Fields to be held in memory for the whole
    duration of current request. This causes high memory usage during sync jobs.

    The data is already written in DB due to branching ChangeDiff. So we can trigger
    the events clearing here to free up memory.

    For performance reasons we don't want to clear the event cache after every object is saved,
    but we also don't want to wait until the end of the sync when syncing large number of objects.
    This class helps to clear events after a defined threshold is reached.

    Without it, the memory requirements are roughly 1,5GB per 10k changes.
    """

    def __init__(self, sender: object, threshold: int = 100) -> None:
        self.sender = sender
        self.threshold = threshold
        self.counter = 0

    def increment(self) -> None:
        """
        Increment the counter and clear events if threshold is reached.

        This should not be called after every instance.snapshot() but instead
        when the instance is processed. This makes sure it's changes are synced
        as a whole. Calling it after every single snapshot() causes issues.
        """
        self.counter += 1
        if self.counter >= self.threshold:
            self.clear()

    def clear(self) -> None:
        logger.debug("Clearing events to reduce memory usage.")
        # This makes sure webhooks are sent properly
        if events := list(events_queue.get().values()):
            flush_events(events)
        # And now the queue can be cleared
        clear_events.send(sender=None)
        self.counter = 0


class IPFabric(object):
    def __init__(self, parameters=None) -> None:
        if parameters:
            self.ipf = IPFClient(**parameters, unloaded=True)
        else:
            self.ipf = IPFClient(
                **settings.PLUGINS_CONFIG["ipfabric_netbox"], unloaded=True
            )
        self.ipf._client.headers[
            "user-agent"
        ] += f'; ipfabric-netbox/{metadata.version("ipfabric-netbox")}'  # noqa: E702

    def get_table_data(self, table, device):
        filter = {"sn": ["eq", device.serial]}
        split = table.split(".")

        if len(split) == 2:
            if split[1] == "serial_ports":
                table = getattr(self.ipf.technology, split[1])
            else:
                tech = getattr(self.ipf.technology, split[0])
                table = getattr(tech, split[1])
        else:
            table = getattr(self.ipf.inventory, split[0])

        columns = self.ipf.get_columns(table.endpoint)

        columns.pop(0)

        columns = [(k, Column()) for k in columns]
        data = table.all(
            filters=filter,
        )
        return data, columns


def make_hashable(obj):
    if isinstance(obj, dict):
        return tuple((k, make_hashable(v)) for k, v in obj.items())
    elif isinstance(obj, list):
        return tuple(make_hashable(x) for x in obj)
    else:
        return obj


class DataRecord:
    """Contains all data required to sync single object to NetBox."""

    def __init__(
        self,
        model_string: str,
        data: dict,
        # These values are filled later as the record is passed down the pipeline
        context: dict | None = None,
        transform_map: "IPFabricTransformMap | None" = None,
    ):
        self.model_string = model_string
        self.data = data
        self.context = context or dict()
        self.transform_map = transform_map

    def __hash__(self):
        if self._hash is None:
            try:
                self._hash = hash(
                    (
                        self.model_string,
                        # Since the dicts are already ordered, it is safe to hash them
                        # .values() are mutable, this is fixed by tuple() to get same hash every time
                        make_hashable(self.data),
                        make_hashable(self.context),
                    )
                )
            except Exception as err:
                raise Exception(f"DATA: {self.data}") from err
        return self._hash

    def __eq__(self, other):
        return isinstance(other, DataRecord) and hash(self) == hash(other)

    # Make sure data and context are sorted by keys when stored to speed up hash calculation
    # This should be safe since they do not contain nested dicts
    @property
    def data(self):
        return self._data

    @data.setter
    def data(self, value):
        self._data = {k: v for k, v in sorted(value.items())}
        self._hash = None  # Invalidate cached hash

    @property
    def context(self):
        return self._context

    @context.setter
    def context(self, value: dict):
        self._context = {k: v for k, v in sorted(value.items())}
        self._hash = None  # Invalidate cached hash

    def reset_context(self) -> None:
        """Clear cached context so it is re-evaluated on the next sync pass.

        The context is computed lazily and then cached on the DataRecord
        (see IPFabricSyncRunner.get_transform_context).  When the same set of
        DataRecords is used for a second sync pass — e.g. assigning
        VirtualChassis.master after Devices have been created — the cached
        context from the first pass would be reused and the master field would
        remain None. Calling this method forces re-evaluation against the
        current DB state.
        """
        self.context = {}


def order_members(members: list[dict], sn_column: str) -> dict[str, list[dict]]:
    """Order stack members to dict, where key is master serial number and values are all members.
    For stack members table we get `memberSn` column for member serial number.
    For VSS chassis table we get `chassisSn` column for member serial number.
    """
    devices = {}

    for member in members:
        # Caution: If the snapshot is created in development mode, the `sn` field is calculated from loopback IP
        # This can be spotted by checking if `sn` is different from `memberSn` for the master device
        # Plus `sn` will be IP of loopback in hex...
        master_serial = member.get("sn")
        if master_serial and member.get(sn_column):
            if master_serial in devices:
                devices[master_serial].append(member)
            else:
                devices[master_serial] = [member]

    return devices


def prepare_devices(
    devices: list[dict], members: dict[str, list[dict]]
) -> tuple[list[dict], list[dict]]:
    """
    Prepare devices list for syncing:
     - incorporating virtual chassis members
     - handle duplicate hostnames
    """

    hostnames = [d["hostname"] for d in devices]
    counter = Counter(hostnames)

    # All devices to be synced
    all_devices = []
    # All virtual chassis to be synced
    virtualchassis = []

    for device in devices[:]:
        if counter[device["hostname"]] > 1:
            device["hostname"] = f"{device['hostname']} - ({device['sn']})"
        if child_members := members.get(device.get("sn")):
            # This device is the VC master, and we're iterating over all it's members
            for child_member in child_members:
                member_id = child_member.get("member") or child_member.get("chassisId")
                member_sn = child_member.get("memberSn") or child_member.get(
                    "chassisSn"
                )
                # There is physically no device with hostname matching the virtual chassis
                # There are only members, so "hostname/1", "hostname/2", etc.
                new_device = deepcopy(device)
                new_device["hostname"] = f"{device['hostname']}/{member_id}"
                new_device["virtual_chassis"] = child_member
                new_device["model"] = child_member.get("pn") or device["model"]
                if device.get("sn") != member_sn:
                    # VC members (non-master) are not in Device table, need to add them as new Device
                    # VSS Chassis have no model in the table, get it from master device
                    new_device["sn"] = member_sn
                else:
                    # Master device, create the virtual chassis based on it
                    virtualchassis.append(child_member)
                all_devices.append(new_device)
            hostnames = [d["hostname"] for d in devices]
            counter = Counter(hostnames)
        else:
            all_devices.append(device)
    return all_devices, virtualchassis


class IPFabricSyncRunner(object):
    def __init__(
        self,
        sync: "IPFabricSync",
        client: IPFClient = None,
        ingestion=None,
        settings: dict = None,
        thread_count: int = 1,
    ) -> None:
        self.client = client
        self.settings = settings
        # We need to get Source for CF in templates, so we preload the FK chain
        self.ingestion = (
            ingestion.__class__.objects.select_related(
                "sync__snapshot_data__source"
            ).get(pk=ingestion.pk)
            if ingestion
            else None
        )
        self.sync = sync
        self.thread_count = max(1, thread_count)
        self.transform_maps = sync.get_transform_maps(sync.parameters.get("groups"))
        if hasattr(self.sync, "logger"):
            self.logger = self.sync.logger

        if self.sync.snapshot_data.status != "loaded":
            raise SyncError("Snapshot not loaded in IP Fabric.")

        # Some objects depend on others being synced first, store errors to avoid duplicates
        # We should store all dependant objects, but it's very hard to do
        # For now store only serial numbers since that is the largest dependency chain
        self.error_serials = set()
        # Lock protecting all shared mutable state written from worker threads:
        #   - self.error_serials
        #   - self.logger.log_data / statistics (via SyncLogging)
        self._lock = threading.Lock()

        self.events_clearer = EventsClearer(sender=self.__class__, threshold=100)

    @staticmethod
    def get_error_serial(context: dict | None, data: dict | None) -> str | None:
        """Get error serial from context or raw data for skipping purposes."""
        context = context or {}
        data = data or {}
        return (
            context.get("sn") or context.get("serial") or serial(data)
            if "sn" in data
            else None
        )

    def create_or_get_sync_issue(
        self,
        exception: Exception,
        ingestion: "IPFabricIngestion",
        message: str = None,
        model_string: str = None,
        context: dict = None,
        data: dict = None,
    ) -> "tuple[bool, IPFabricIngestionIssue]":
        """
        Helper function to handle sync errors and create IPFabricIngestionIssue if needed.
        """
        if isinstance(exception, JobTimeoutException):
            # These should not be caught, so let's just process it and re-raise
            raise

        context = context or {}

        error_serial = self.get_error_serial(context, data)
        # Ignore models that do not have any dependencies by serial number
        if error_serial and model_string not in ["ipam.ipaddress", "dcim.macaddress"]:
            with self._lock:
                self.error_serials.add(error_serial)

        # TODO: This is to prevent circular import issues, clean it up later.
        from ..models import IPFabricIngestionIssue

        if not hasattr(exception, "issue_id") or not exception.issue_id:
            issue = IPFabricIngestionIssue.objects.create(
                ingestion=ingestion,
                exception=exception.__class__.__name__,
                message=message or getattr(exception, "message", str(exception)),
                model=model_string,
                coalesce_fields={
                    k: v for k, v in context.items() if k not in ["defaults"]
                },
                defaults=context.get("defaults", dict()),
                raw_data=data or dict(),
            )
            if hasattr(exception, "issue_id"):
                exception.issue_id = issue.id
            return True, issue
        else:
            issue = IPFabricIngestionIssue.objects.get(id=exception.issue_id)
            return False, issue

    @staticmethod
    def handle_errors(func: Callable):
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except JobTimeoutException as err:
                self = args[0]

                # JobTimeoutException abruptly closes DB connection, leaving it in inconsistent state
                # Let's close it first before doing anything else to prevent any weird issues when handling the timeout
                # Since we are in RQ Worker, we should not be able to touch other connections so we can close them all
                # We cannot get DB connection name from self.get_db_connection_name() since it does DB call, causing race condition
                for conn in connections.all():
                    if conn.connection:
                        conn.close()

                # TODO: This is to prevent circular import issues, clean it up later.
                from ..models import IPFabricIngestionIssue

                # We can have nested decorators, make sure to log it only once.
                if not IPFabricIngestionIssue.objects.filter(
                    ingestion=self.ingestion, exception=err.__class__.__name__
                ).exists():
                    IPFabricIngestionIssue.objects.create(
                        ingestion=self.ingestion,
                        exception=err.__class__.__name__,
                        message=str(err),
                    )
                    logger.error(err, exc_info=True)
                # Now we can re-raise to handle it in IPFabricSync.sync()
                raise
            except Exception as err:
                # Log the error to logger outside of job - console/file
                logger.error(err, exc_info=True)
                if hasattr(err, "issue_id") and err.issue_id:
                    # The error is already logged to user, no need to log it again
                    return None
                # Logging section for logs inside job - facing user
                self = args[0]
                if isinstance(err, SearchError):
                    if self.settings.get(err.model_string):
                        self.logger.log_failure(
                            f"Aborting syncing `{err.model_string}` instance due to above error, please check your transform maps and/or existing data.",
                            obj=self.sync,
                        )
                    else:
                        self.logger.log_failure(
                            f"Syncing `{err.model_string}` is disabled in settings, but hit above error trying to find the correct item. Please check your transform maps and/or existing data.",
                            obj=self.sync,
                        )
                if isinstance(err, IPAddressDuplicateError):
                    self.logger.log_warning(
                        f"IP Address `{err.data.get('address')}` already exists in `{err.model_string}` with coalesce fields: `{err.coalesce_fields}`. Please check your transform maps and/or existing data.",
                        obj=self.sync,
                    )
                else:
                    self.logger.log_failure(
                        f"Syncing failed with: `{err}`. See above error for more details.",
                        obj=self.sync,
                    )
                # Make sure the whole sync is failed when we encounter error
                self.sync.status = IPFabricSyncStatusChoices.FAILED
                return None

        return wrapper

    @cache
    def get_db_connection_name(self) -> str:
        connection_name = None
        if self.ingestion:
            connection_name = self.ingestion.branch.connection_name
        return connection_name

    def get_transform_context(self, record: DataRecord) -> DataRecord:
        if record.context:
            # Context might already be calculated, so we can skip it to save some time.
            return record
        # Inject the ingestion object so templates (e.g. cf_ipfabric_source /
        # cf_ipfabric_ingestion) can derive whatever they need from it.
        record.data = {
            **record.data,
            "ingestion": self.ingestion,
        }
        if not record.transform_map:
            raise SystemError(f"No transform map available for {record.model_string}")
        try:
            record.context = record.transform_map.get_context(
                record.data, connection_name=self.get_db_connection_name()
            )
        except Exception as err:
            message = f"Error getting context for `{record.model_string}`."
            logger.info(f"{type(err)}, {dir(err)}")

            def _construct_query_cmd(e) -> str:
                """Helper function to construct used ORM query for logging purposes."""
                model = e.__class__.__qualname__.split(".", maxsplit=1)[0]
                return f"{model}({', '.join(f'{k}={v}' for k, v in record.context.items() if k != 'defaults')})"

            # Check if exception has enriched metadata from build_relationships or render
            field_name = getattr(err, "field_name", None)
            field_type = getattr(err, "field_type", None)
            if field_name and field_type:
                # We have enriched metadata - format message accordingly
                if getattr(err, "is_template_error", False):
                    message += (
                        f" Error rendering template for {field_type} `{field_name}`."
                    )
                elif isinstance(err, ObjectDoesNotExist):
                    message += f" Error querying for {field_type} `{field_name}` with evaluated query {_construct_query_cmd(err)} from template in transform maps could not find related object."
                elif isinstance(err, MultipleObjectsReturned):
                    message += f" Error querying for {field_type} `{field_name}` with evaluated query {_construct_query_cmd(err)} from template in transform maps returned multiple objects, the template is not strict enough."
                else:
                    # Other errors - just include field info
                    message += f" Error processing {field_type} `{field_name}`."
            else:
                # Fallback to original behavior for errors without metadata
                if isinstance(err, ObjectDoesNotExist):
                    message += f" Evaluated query {_construct_query_cmd(err)} from template in transform maps could not find related object."
                elif isinstance(err, MultipleObjectsReturned):
                    message += f" Evaluated query {_construct_query_cmd(err)} from template in transform maps returned multiple objects, the template is not strict enough."

            _, issue = self.create_or_get_sync_issue(
                exception=err,
                ingestion=self.ingestion,
                message=message,
                model_string=record.model_string,
                data=record.data,
            )
            raise SearchError(
                message=message,
                data=record.data,
                model_string=record.model_string,
                issue_id=issue.pk,
            ) from err

        return record

    def get_model_or_update(self, record: DataRecord) -> ModelTypeVar | None:
        # First check if there are any previous errors linked to this object
        error_serial = self.get_error_serial(record.context, record.data)
        if error_serial:
            with self._lock:
                serial_has_error = error_serial in self.error_serials
            if serial_has_error:
                # We want to raise it as exception so it's shown in ingestion issues but can be filtered out.
                exception = RequiredDependencyFailedSkip(
                    message=f"Skipping syncing of `{record.model_string}` with serial `{error_serial}` due to previous errors.",
                    model_string=record.model_string,
                    context=record.context,
                    data=record.data,
                )
                _, issue = self.create_or_get_sync_issue(
                    exception=exception,
                    ingestion=self.ingestion,
                    model_string=record.model_string,
                    context=record.context,
                    data=record.data,
                )
                raise exception
        record = self.get_transform_context(record)
        queryset = record.transform_map.target_model.model_class().objects
        model_settings = self.settings.get(f"{record.model_string}", False)

        obj = None
        try:
            connection_name = self.get_db_connection_name()
            if model_settings:
                logger.info(f"Creating {record.model_string}")
                obj = record.transform_map.update_or_create_instance(
                    context=record.context,
                    tags=self.sync.tags.all(),
                    connection_name=connection_name,
                )
            else:
                logger.info(f"Getting {record.model_string}")
                record.context.pop("defaults", None)
                obj = queryset.using(connection_name).get(**record.context)
        except queryset.model.DoesNotExist as err:
            message = f"Instance of `{record.model_string}` not found."
            _, issue = self.create_or_get_sync_issue(
                exception=err,
                ingestion=self.ingestion,
                message=message,
                model_string=record.model_string,
                context=record.context,
                data=record.data,
            )
            raise SearchError(
                message=message,
                model_string=record.model_string,
                context=record.context,
                data=record.data,
                issue_id=issue.pk,
            ) from err
        except queryset.model.MultipleObjectsReturned as err:
            message = f"Multiple instances of `{record.model_string}` found."
            _, issue = self.create_or_get_sync_issue(
                exception=err,
                ingestion=self.ingestion,
                message=message,
                model_string=record.model_string,
                context=record.context,
                data=record.data,
            )
            raise SearchError(
                message=message,
                model_string=record.model_string,
                context=record.context,
                data=record.data,
                issue_id=issue.pk,
            ) from err
        except Exception as err:
            _, issue = self.create_or_get_sync_issue(
                exception=err,
                ingestion=self.ingestion,
                model_string=record.model_string,
                context=record.context,
                data=record.data,
            )
            raise SyncDataError(
                model_string=record.model_string,
                context=record.context,
                data=record.data,
                issue_id=issue.pk,
            ) from err

        return obj

    def collect_data(self):
        # Importing here to avoid circular import
        from ..models import IPFabricEndpoint

        try:
            self.logger.log_info(
                "Collecting information from IP Fabric",
                obj=self.sync.snapshot_data.source,
            )
            data = {}
            # TODO: Replace endpoint value in SnapshotData with Endpoint link
            if self.sync.snapshot_data.source.type == IPFabricSourceTypeChoices.REMOTE:
                # This requires data already pushed to NetBox by user, does not connect to IPF directly
                self.logger.log_info(
                    "Remote collector checking for snapshot data.", obj=self.sync
                )
                if not self.sync.snapshot_data.ipf_data.count() > 0:
                    raise SyncError(
                        "No snapshot data available. This is a remote sync. Push data to NetBox first."
                    )
                for endpoint in IPFabricEndpoint.objects.all():
                    data[endpoint.endpoint] = list(
                        self.sync.snapshot_data.ipf_data.filter(
                            type=endpoint.endpoint
                        ).values_list("data", flat=True)
                    )
            else:
                # This pulls data directly from IP Fabric instance
                # TODO: If certain columns are not enabled in IPF, sync will fail in odd ways
                #  For example missing `nameOriginal` on Interface makes it unable to sync IPs
                #  When pulling from endpoint, make sure to specify columns according to transform maps
                #  This can be pulled using logic in IPFabricTransformMap.strip_source_data()
                # TODO: Also cache it to improve performance, no need to regex every time!
                self.logger.log_info(
                    "Local collector being used for snapshot data.", obj=self.sync
                )

                # Get enabled endpoints from the sync
                enabled_endpoints = self.sync.get_enabled_endpoints()

                transform_maps = self.sync.get_transform_maps()
                for endpoint in enabled_endpoints:
                    # FIXME: Dirty hack to sync VSS, remove when IN-68 is getting done
                    if endpoint.endpoint == IPFabricEndpointChoices.VSS_CHASSIS:
                        tms_for_endpoint = transform_maps.filter(
                            source_endpoint__endpoint=IPFabricEndpointChoices.VIRTUALCHASSIS
                        )
                    else:
                        tms_for_endpoint = transform_maps.filter(
                            source_endpoint=endpoint
                        )

                    if not tms_for_endpoint.exists():
                        raise SyncError(
                            f"No transform map found for endpoint `{endpoint.endpoint}`."
                        )

                    filters = endpoint.combine_filters(sync=self.sync)
                    logger.debug(
                        f"Collecting data from endpoint: `{endpoint.endpoint}` using filter `{json.dumps(filters)}`."
                    )
                    data[endpoint.endpoint] = self.client.fetch_all(
                        url=endpoint.endpoint,
                        snapshot_id=self.settings["snapshot_id"],
                        filters=filters,
                    )
                    self.logger.log_info(
                        f"Collected {len(data[endpoint.endpoint])} items from endpoint `{endpoint.endpoint}`.",
                        obj=self.sync.snapshot_data.source,
                    )
        except Exception as e:
            self.logger.log_failure(
                f"Error collecting data from IP Fabric: {e}", obj=self.sync
            )
            raise SyncError(f"Error collecting data from IP Fabric: {e}")
        return data

    @cache
    def get_transform_map(
        self, app: str, model: str, endpoint: str = None
    ) -> "IPFabricTransformMap":
        """Get transform map for given app and model. Cached to improve performance."""
        filter_kwargs = {
            "target_model__app_label": app,
            "target_model__model": model,
        }
        if endpoint:
            filter_kwargs["source_endpoint__endpoint"] = endpoint
        return self.transform_maps.get(**filter_kwargs)

    def create_new_data_records(
        self,
        app: str,
        model: str,
        data: list[dict] | dict[str, list[dict]],
        endpoint: str = None,
    ) -> set[DataRecord]:
        """Create DataRecord objects for given app, model and data list if enabled in settings."""
        if not self.sync.parameters.get(f"{app}.{model}"):
            return set()
        if isinstance(data, dict):
            # Data is either already list or full data where we have to choose the list
            transform_map = self.get_transform_map(
                app=app, model=model, endpoint=endpoint
            )
            data = data.get(transform_map.source_endpoint.endpoint, [])
        return set(
            self.create_new_data_record(
                app=app, model=model, data=item, endpoint=endpoint
            )
            for item in data
        )

    def create_new_data_record(
        self, app: str, model: str, data: dict, endpoint: str = None
    ) -> DataRecord:
        """Extract only relevant source data according to transform map configuration."""
        transform_map = self.get_transform_map(app=app, model=model, endpoint=endpoint)
        model_string = f"{app}.{model}"
        try:
            source_data = transform_map.strip_source_data(data)
        except KeyError as err:
            raise SyncError(
                f"Missing key column {err.args[0]} in source data when preparing data for {model_string}."
            ) from err
        return DataRecord(
            model_string=model_string, data=source_data, transform_map=transform_map
        )

    def _get_custom_preprocessor(
        self, model_string: str, endpoint: str
    ) -> Callable | None:
        """
        Get custom preprocessing function for a model+endpoint combination if one exists.

        Args:
            model_string: The model in 'app.model' format (e.g., 'dcim.virtualchassis')
            endpoint: The source endpoint (e.g., '/technology/platforms/stack/members')

        Returns:
            Callable preprocessor function or None if no custom preprocessor exists
        """
        preprocessors = {
            # Devices and related models processed together (from /inventory/devices)
            ("dcim.device", IPFabricEndpointChoices.DEVICES): self._preprocess_devices,
            (
                "dcim.manufacturer",
                IPFabricEndpointChoices.DEVICES,
            ): self._preprocess_devices,
            (
                "dcim.devicetype",
                IPFabricEndpointChoices.DEVICES,
            ): self._preprocess_devices,
            (
                "dcim.platform",
                IPFabricEndpointChoices.DEVICES,
            ): self._preprocess_devices,
            (
                "dcim.devicerole",
                IPFabricEndpointChoices.DEVICES,
            ): self._preprocess_devices,
            # Virtual chassis from stack members
            (
                "dcim.virtualchassis",
                IPFabricEndpointChoices.VIRTUALCHASSIS,
            ): self._preprocess_devices,
            # Interfaces and MAC addresses processed together
            (
                "dcim.interface",
                IPFabricEndpointChoices.INTERFACES,
            ): self._preprocess_interfaces,
            (
                "dcim.macaddress",
                IPFabricEndpointChoices.INTERFACES,
            ): self._preprocess_interfaces,
            # IP addresses need interface data
            (
                "ipam.ipaddress",
                IPFabricEndpointChoices.IPADDRESSES,
            ): self._preprocess_ipaddresses,
        }

        return preprocessors.get((model_string, endpoint))

    def _preprocess_devices(self, data: dict, records: dict) -> None:
        """Custom preprocessing for devices - single iteration for efficiency."""
        if not (
            self.sync.parameters.get("dcim.device")
            or self.sync.parameters.get("dcim.virtualchassis")
            or self.sync.parameters.get("dcim.interface")
            or self.sync.parameters.get("dcim.macaddress")
            or self.sync.parameters.get("ipam.ipaddress")
        ):
            return

        # Prepare virtual chassis members once
        members = {}
        if self.sync.parameters.get("dcim.virtualchassis"):
            self.logger.log_info("Preparing virtual chassis members", obj=self.sync)
            members = order_members(
                data.get(IPFabricEndpointChoices.VIRTUALCHASSIS, []), "memberSn"
            )
            members.update(
                order_members(
                    data.get(IPFabricEndpointChoices.VSS_CHASSIS, []), "chassisSn"
                )
            )

        self.logger.log_info("Preparing devices", obj=self.sync)
        devices, virtualchassis = prepare_devices(
            data.get(IPFabricEndpointChoices.DEVICES, []), members
        )

        device_primary_ips = {}

        # Single iteration creating records for all related models
        for device in devices:
            if self.sync.parameters.get("dcim.manufacturer"):
                records["dcim.manufacturer"].add(
                    self.create_new_data_record(
                        app="dcim", model="manufacturer", data=device
                    )
                )

            if self.sync.parameters.get("dcim.devicetype"):
                records["dcim.devicetype"].add(
                    self.create_new_data_record(
                        app="dcim", model="devicetype", data=device
                    )
                )

            if self.sync.parameters.get("dcim.platform"):
                records["dcim.platform"].add(
                    self.create_new_data_record(
                        app="dcim", model="platform", data=device
                    )
                )

            if self.sync.parameters.get("dcim.devicerole"):
                records["dcim.devicerole"].add(
                    self.create_new_data_record(
                        app="dcim", model="devicerole", data=device
                    )
                )

            if "virtual_chassis" not in device:
                device["virtual_chassis"] = None

            if self.sync.parameters.get("dcim.device"):
                records["dcim.device"].add(
                    self.create_new_data_record(app="dcim", model="device", data=device)
                )

            device_primary_ips[device.get("sn")] = device.get("loginIp") or device.get(
                "loginIpv4"
            )

        self._device_primary_ips = device_primary_ips
        records["dcim.virtualchassis"] = self.create_new_data_records(
            app="dcim", model="virtualchassis", data=virtualchassis
        )

    def _preprocess_interfaces(self, data: dict, records: dict) -> None:
        """Custom preprocessing for interfaces and MAC addresses."""
        if not (
            self.sync.parameters.get("dcim.interface")
            or self.sync.parameters.get("dcim.macaddress")
            or self.sync.parameters.get("ipam.ipaddress")
        ):
            return

        if not hasattr(self, "_device_primary_ips"):
            self._preprocess_devices(data=data, records=records)

        interface_key = "nameOriginal"
        try:
            int_transform_map = self.get_transform_map(app="dcim", model="interface")
            int_name_field_map = int_transform_map.field_maps.get(target_field="name")
            interface_key = int_name_field_map.source_field
        except Exception as e:
            self.logger.log_failure(
                f"Error collecting transform map info: {e}", obj=self.sync
            )
            raise SyncError(f"Error collecting source column name for interface: {e}")

        # Store human-readable interface names to use them later for IP Addresses
        readable_int_names = {}
        self.logger.log_info("Preparing Interfaces", obj=self.sync)

        for interface in data.get(IPFabricEndpointChoices.INTERFACES, []):
            if self.sync.parameters.get("dcim.interface"):
                interface_record = self.create_new_data_record(
                    app="dcim", model="interface", data=interface
                )
                interface_record.data["loginIp"] = self._device_primary_ips.get(
                    interface.get("sn")
                )
                records["dcim.interface"].add(interface_record)

            readable_int_names[
                f"{interface.get('sn')}_{interface.get('intName')}"
            ] = interface.get(interface_key)

            if self.sync.parameters.get("dcim.macaddress"):
                records["dcim.macaddress"].add(
                    self.create_new_data_record(
                        app="dcim", model="macaddress", data=interface
                    )
                )

        self._readable_int_names = readable_int_names

    def _preprocess_ipaddresses(self, data: dict, records: dict) -> None:
        """Custom preprocessing for IP addresses."""
        if not self.sync.parameters.get("ipam.ipaddress"):
            return

        if not hasattr(self, "_readable_int_names"):
            self._preprocess_interfaces(data=data, records=records)

        self.logger.log_info("Preparing IP Addresses", obj=self.sync)
        for ip in data.get("/technology/addressing/managed-ip/ipv4", []):
            ip["nameOriginal"] = self._readable_int_names.get(
                f"{ip.get('sn')}_{ip.get('intName')}"
            )
            if not ip["nameOriginal"]:
                continue

            ipaddress_record = self.create_new_data_record(
                app="ipam", model="ipaddress", data=ip
            )
            ipaddress_record.data["is_primary"] = ip.get(
                "sn"
            ) in self._device_primary_ips and self._device_primary_ips.get(
                ip.get("sn")
            ) == ipaddress_record.data.get(
                "ip"
            )
            records["ipam.ipaddress"].add(ipaddress_record)

    def preprocess_data(self, data: dict) -> dict[str, set[DataRecord]]:
        """
        Preprocess data using hierarchical transform map order.

        Iterates over transform maps in dependency order (parents before children),
        using custom preprocessing functions where available or direct record creation.
        Each transform map is processed exactly once, avoiding duplicate iterations.
        """
        # Get transform maps in hierarchical order (now returns transform maps directly!)
        group_ids = self.sync.parameters.get("groups", [])
        transform_maps_hierarchy = self.sync.get_model_hierarchy(group_ids=group_ids)

        # Initialize records dict for all models (extract unique model strings)
        records = {}
        seen_models = set()
        for transform_map in transform_maps_hierarchy:
            model_string = f"{transform_map.target_model.app_label}.{transform_map.target_model.model}"
            if model_string not in seen_models:
                records[model_string] = set()
                seen_models.add(model_string)

        # Track which custom preprocessors have run (to avoid duplication)
        # Use (preprocessor, endpoint) tuple to allow same preprocessor for different endpoints
        preprocessors_run = set()

        # Process each transform map in hierarchical order
        for transform_map in transform_maps_hierarchy:
            app = transform_map.target_model.app_label
            model = transform_map.target_model.model
            model_string = f"{app}.{model}"
            endpoint = transform_map.source_endpoint.endpoint

            # Skip if not enabled in sync parameters
            if not self.sync.parameters.get(model_string):
                continue

            # Check for custom preprocessor (pass endpoint for endpoint-specific preprocessing)
            preprocessor = self._get_custom_preprocessor(model_string, endpoint)

            # Create unique key combining preprocessor function and endpoint
            preprocessor_key = preprocessor if preprocessor else None

            if preprocessor and preprocessor_key not in preprocessors_run:
                # Run custom preprocessor (may populate multiple models)
                self.logger.log_info(
                    f"Running custom preprocessor for `{model_string}` from `{endpoint}`",
                    obj=self.sync,
                )
                preprocessor(data, records)
                preprocessors_run.add(preprocessor_key)
            elif not preprocessor:
                # Standard processing: use the transform map directly (no need to filter!)
                self.logger.log_info(
                    f"Preparing `{model_string}` from `{endpoint}`", obj=self.sync
                )
                records[model_string].update(
                    self.create_new_data_records(
                        app=app, model=model, data=data, endpoint=endpoint
                    )
                )

        # Log statistics
        for model_string, records_set in records.items():
            if self.settings.get(model_string) and len(records_set):
                self.logger.init_statistics(model_string, len(records_set))
                self.logger.log_info(
                    f"Prepared {len(records_set)} items for `{model_string}` to be synced.",
                    obj=self.sync,
                )

        return records

    @handle_errors
    def sync_model(
        self,
        record: DataRecord,
        stats: bool = True,
    ) -> ModelTypeVar | None:
        """Sync a single item to NetBox."""
        if not record.data:
            return None

        instance = self.get_model_or_update(record)

        # Only log when we successfully synced the item and asked for it
        if stats and instance:
            self.logger.increment_statistics(model_string=record.model_string)

        return instance

    # Minimum number of records per chunk. Splitting below this threshold
    # creates more thread overhead than it saves — each thread needs to open
    # a DB connection, set up a request context, etc. With fewer records than
    # this per chunk, single-threaded execution is faster.
    _MIN_CHUNK_SIZE = 10

    def _split_into_chunks(self, items: set[DataRecord]) -> list[list[DataRecord]]:
        """
        Split a set of DataRecords into roughly equal chunks — one per thread.

        Clamps the effective worker count so we never create more chunks than
        items, and also enforces a minimum chunk size (_MIN_CHUNK_SIZE) so that
        tiny record sets are not split into many single-item chunks where thread
        overhead outweighs the parallelism benefit.
        """
        items_list = list(items)
        n = len(items_list)
        # Never use more threads than we have items
        effective_workers = min(self.thread_count, n)
        if effective_workers <= 1:
            return [items_list]
        # Ensure each chunk has at least _MIN_CHUNK_SIZE records.
        # e.g. 3 records with 20 threads → chunk_size=10 → max_workers=1 → single-threaded
        chunk_size = max(math.ceil(n / effective_workers), self._MIN_CHUNK_SIZE)
        # Recalculate actual worker count based on the enforced chunk size
        actual_workers = math.ceil(n / chunk_size)
        if actual_workers <= 1:
            return [items_list]
        return [
            items_list[i : i + chunk_size]  # noqa: E203
            for i in range(0, n, chunk_size)
        ]

    @cache
    def _requires_single_thread(self, model_string: str) -> bool:
        """
        Return True if this model must be synced single-threaded.

        Django MPTT restructures the whole tree on every insert/update by issuing:

            UPDATE ... SET tree_id = tree_id + 1 WHERE tree_id > N

        With multiple concurrent writers, overlapping WHERE tree_id > N ranges
        produce row-level lock cycles → deadlock. NetBox's own API serialises
        writes to MPTT-backed models via PostgreSQL advisory locks
        (MPTTLockedMixin). We take the equivalent — and simpler — approach:
        always sync MPTT-backed models single-threaded.

        Detection is fully dynamic: we resolve the model class at runtime and
        check for the '_mptt_meta' attribute that Django-MPTT's metaclass sets
        on every MPTTModel subclass. This means any new MPTT model added to
        NetBox (or to the sync) is handled automatically without code changes.

        If django-treebeard (MP_Node / NS_Node) is ever introduced, add an
        analogous check here.
        """
        from django.apps import apps

        app_label, model_name = model_string.split(".")
        try:
            model_class = apps.get_model(app_label, model_name)
        except LookupError:
            return False
        return hasattr(model_class, "_mptt_meta")

    def _sync_chunk(
        self,
        chunk: list[DataRecord],
        ingestion: "IPFabricIngestion" = None,
        stats: bool = True,
    ) -> None:
        """
        Process a single chunk of DataRecords inside a worker thread.

        Each thread calls close_old_connections() so Django opens a fresh,
        thread-local DB connection rather than sharing the caller's connection.
        A thread-local EventsClearer is used so the events queue ContextVar
        (already isolated per-thread) is flushed properly within this thread.
        """
        close_old_connections()
        local_clearer = EventsClearer(sender=self.__class__, threshold=100)
        try:
            for item in chunk:
                self.sync_model(record=item, stats=stats)
                local_clearer.increment()
            local_clearer.clear()
        finally:
            connections.close_all()

    def _sync_devices_chunk(
        self,
        chunk: list[DataRecord],
        ingestion: "IPFabricIngestion" = None,
    ) -> None:
        """Worker-thread body for syncing a chunk of Device records."""
        close_old_connections()
        local_clearer = EventsClearer(sender=self.__class__, threshold=100)
        try:
            for device in chunk:
                self.sync_device(
                    device, ingestion=ingestion, events_clearer=local_clearer
                )
                local_clearer.increment()
            local_clearer.clear()
        finally:
            connections.close_all()

    def _sync_ipaddresses_chunk(self, chunk: list[DataRecord]) -> None:
        """Worker-thread body for syncing a chunk of IPAddress records."""
        close_old_connections()
        local_clearer = EventsClearer(sender=self.__class__, threshold=100)
        try:
            for ip_address in chunk:
                self.sync_ipaddress(ip_address, events_clearer=local_clearer)
            local_clearer.clear()
        finally:
            connections.close_all()

    def sync_items(
        self,
        items: set[DataRecord],
        ingestion: "IPFabricIngestion" = None,
        stats: bool = True,
    ) -> None:
        """Sync a set of items to NetBox, distributing them across a thread pool."""
        if not items:
            return

        model_string = next(iter(items)).model_string
        if not self.settings.get(model_string):
            self.logger.log_info(
                f"Did not ask to sync {model_string}s, skipping.", obj=self.sync
            )
            return

        if self._requires_single_thread(model_string):
            # MPTT-backed models must be written single-threaded: concurrent
            # inserts deadlock via overlapping tree_id range updates. See
            # _requires_single_thread() for full explanation.
            self.logger.log_info(
                f"Syncing `{model_string}` single-threaded (MPTT model, {len(items)} records).",
                obj=self.sync,
            )
            chunks = [list(items)]
        else:
            chunks = self._split_into_chunks(items)

        if len(chunks) == 1:
            # Single-threaded fast path — avoids ThreadPoolExecutor overhead,
            # also used for MPTT models that must not be written concurrently.
            for item in chunks[0]:
                self.sync_model(record=item, stats=stats)
                self.events_clearer.increment()
            return

        self.logger.log_info(
            f"Syncing `{model_string}` using {len(chunks)} threads ({len(items)} records).",
            obj=self.sync,
        )
        # Capture the full ContextVar context (includes active_branch from the
        # branching plugin) so every worker thread runs with the same branch set.
        # Without this, worker threads have no branch context and Django's
        # full_clean() GFK validation queries hit the default DB instead of the
        # branch DB, causing spurious "Related object not found" errors.
        # A fresh copy_context() is made per submission — ctx.run() is not
        # re-entrant and cannot be shared across concurrent threads.
        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
            futures = [
                executor.submit(
                    contextvars.copy_context().run,
                    self._sync_chunk,
                    chunk,
                    ingestion,
                    stats,
                )
                for chunk in chunks
            ]
            futures_wait(futures)
        # Re-raise first exception from any worker, if any
        for future in futures:
            future.result()

    @handle_errors
    def sync_ipaddress(
        self,
        ip_address: DataRecord,
        events_clearer: "EventsClearer | None" = None,
    ) -> "IPAddress | None":
        """Sync a single IP Address to NetBox, separated to use @handle_errors."""
        if events_clearer is None:
            events_clearer = self.events_clearer
        connection_name = self.get_db_connection_name()

        if ip_address.data.get("is_primary"):
            # Remove primary IP from any *other* device that currently holds it.
            # It cannot be done using hooks since there is no pre_clean, and it
            # fails on full_clean().
            try:
                ipv4_address = render_jinja2(
                    ip_address.transform_map.field_maps.get(
                        target_field="address"
                    ).template,
                    {"object": ip_address.data},
                )
                other_device = (
                    Device.objects.using(connection_name)
                    .exclude(serial=serial(ip_address.data))
                    .get(primary_ip4__address=ipv4_address)
                )
                other_device.snapshot()
                other_device.primary_ip4 = None
                other_device.save(using=connection_name)
            except Device.DoesNotExist:
                # There is no other device with this IP as primary, all good
                pass
            except Exception as err:
                # The transform maps might be changed, and we fail to resolve the template
                # Make sure this does not crash the sync and is handled gracefully
                _, issue = self.create_or_get_sync_issue(
                    exception=err,
                    ingestion=self.ingestion,
                    message="Error removing primary IP from other device.",
                    model_string=ip_address.model_string,
                    data=ip_address.data,
                )
                events_clearer.increment()
                raise IPAddressPrimaryRemovalError(
                    data=ip_address.data,
                    model_string=ip_address.model_string,
                    issue_id=issue.pk,
                ) from err

        ip_address_obj: "IPAddress | None" = self.sync_model(record=ip_address)
        if ip_address_obj is None or ip_address_obj.assigned_object is None:
            events_clearer.increment()
            return

        parent_device = ip_address_obj.assigned_object.parent_object

        # Now assign this IP as primary to the parent device, if not assigned yet or assigned to different IP
        if ip_address.data.get("is_primary") and (
            not parent_device.primary_ip4
            or parent_device.primary_ip4.pk != ip_address_obj.pk
        ):
            try:
                parent_device.snapshot()
                parent_device.primary_ip4 = ip_address_obj
                parent_device.save(using=connection_name)
            except Exception as err:
                _, issue = self.create_or_get_sync_issue(
                    exception=err,
                    ingestion=self.ingestion,
                    message="Error assigning primary IP to device.",
                    model_string=ip_address.model_string,
                    data=ip_address.data,
                )
                events_clearer.increment()
                raise IPAddressPrimaryAssignmentError(
                    data=ip_address.data,
                    model_string=ip_address.model_string,
                    issue_id=issue.pk,
                ) from err
        events_clearer.increment()

    @handle_errors
    def sync_ip_addresses(self, ip_addresses: set[DataRecord]) -> None:
        """
        We cannot assign primary IP in signals since IPAddress does not
        contain information whether it is primary or not. And it must be done
        on Device object, so cannot be done via Transform Maps yet since that
        would require another Transform Map for Device.
        So we need to do it manually here.

        Cleaning events queue happens during each cycle to make sure all required
        operations (primary IP assignment) happen during the same batch.
        """
        if not self.settings.get("ipam.ipaddress"):
            self.logger.log_info(
                "Did not ask to sync ipaddresses, skipping.", obj=self.sync
            )
            return

        chunks = self._split_into_chunks(ip_addresses)
        if len(chunks) == 1:
            for ip_address in chunks[0]:
                self.sync_ipaddress(ip_address)
            return

        self.logger.log_info(
            f"Syncing `ipam.ipaddress` using {len(chunks)} threads ({len(ip_addresses)} records).",
            obj=self.sync,
        )
        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
            futures = [
                executor.submit(
                    contextvars.copy_context().run, self._sync_ipaddresses_chunk, chunk
                )
                for chunk in chunks
            ]
            futures_wait(futures)
        for future in futures:
            future.result()

    @handle_errors
    def sync_device(
        self,
        device: DataRecord,
        ingestion: "IPFabricIngestion" = None,
        events_clearer: "EventsClearer | None" = None,
    ) -> Device | None:
        """
        Sync a single Device to NetBox.

        Before syncing, handle VirtualChassis master removal logic that cannot work
        in signal due to missing pre_clean signal.
        """
        if events_clearer is None:
            events_clearer = self.events_clearer
        connection_name = self.get_db_connection_name()

        # First, handle VirtualChassis master removal BEFORE syncing the device
        # Get the device context to determine the target virtual_chassis
        try:
            device = self.get_transform_context(device)
        except Exception:
            # If we can't get context, proceed with normal sync
            pass

        # Check if device will be assigned to a VirtualChassis or removed from one
        target_vc = device.context.get("defaults", {}).get(
            "virtual_chassis"
        ) or device.context.get("virtual_chassis")

        # We need to handle VC master removal in two cases:
        # 1. When moving device to a different VC (target_vc is set)
        # 2. When removing device from VC completely (target_vc is None)
        try:
            existing_device = Device.objects.using(connection_name).get(
                serial=device.context.get("serial")
            )

            # Get target VC object if it exists
            target_vc_obj = None
            if target_vc:
                with contextlib.suppress(VirtualChassis.DoesNotExist):
                    target_vc_obj = VirtualChassis.objects.using(connection_name).get(
                        pk=target_vc.pk
                    )

            # Find all VCs where this device is master
            master_vc = VirtualChassis.objects.using(connection_name).get(
                master=existing_device
            )
            # Remove device as master if:
            # - Moving to a different VC (vc != target_vc_obj)
            # - OR removing from VC completely (target_vc is None)
            if (target_vc_obj and master_vc != target_vc_obj) or target_vc is None:
                master_vc.snapshot()
                master_vc.master = None
                master_vc.save(using=connection_name)
        except Device.DoesNotExist:
            # There is no Device yet, no need to remove it from other VC
            pass
        except VirtualChassis.DoesNotExist:
            # Not assigned to another VirtualChassis, cannot be a master
            pass
        except Exception as err:
            _, issue = self.create_or_get_sync_issue(
                exception=err,
                ingestion=self.ingestion,
                message="Error removing device as master from VirtualChassis.",
                model_string=device.model_string,
                context=device.context,
                data=device.data,
            )
            events_clearer.increment()
            raise SyncDataError(
                data=device.data,
                model_string=device.model_string,
                issue_id=issue.pk,
            ) from err

        # Now sync the device
        device_obj: Device | None = self.sync_model(record=device)

        events_clearer.increment()
        return device_obj

    def sync_devices(
        self,
        devices: set[DataRecord],
        ingestion: "IPFabricIngestion" = None,
    ) -> None:
        """
        Sync devices to NetBox with custom VirtualChassis master removal logic.

        We cannot rely on a signal since there is no pre_clean signal, and we
        need to handle it before the device is created/updated.
        """
        if not self.settings.get("dcim.device"):
            self.logger.log_info(
                "Did not ask to sync devices, skipping.", obj=self.sync
            )
            return

        chunks = self._split_into_chunks(devices)
        if len(chunks) == 1:
            for device in chunks[0]:
                self.sync_device(device, ingestion=ingestion)
            return

        self.logger.log_info(
            f"Syncing `dcim.device` using {len(chunks)} threads ({len(devices)} records).",
            obj=self.sync,
        )
        with ThreadPoolExecutor(max_workers=len(chunks)) as executor:
            futures = [
                executor.submit(
                    contextvars.copy_context().run,
                    self._sync_devices_chunk,
                    chunk,
                    ingestion,
                )
                for chunk in chunks
            ]
            futures_wait(futures)
        for future in futures:
            future.result()

    def collect_and_sync(self, ingestion=None) -> None:
        self.logger.log_info("Starting data collection.", obj=self.sync)
        data = self.collect_data()
        self.logger.log_info("Starting to prepare items.", obj=self.sync)
        records = self.preprocess_data(data=data)

        self.logger.log_info("Starting data sync.", obj=self.sync)
        try:
            # This signal does not call for snapshot(), causing issue with branching plugin
            if sync_cached_scope_fields is not None:
                signals.post_save.disconnect(sync_cached_scope_fields, sender=Site)
            self.sync_items(items=records["dcim.site"], ingestion=ingestion)
        finally:
            if sync_cached_scope_fields is not None:
                signals.post_save.connect(sync_cached_scope_fields, sender=Site)
        self.sync_items(items=records["dcim.manufacturer"])
        self.sync_items(items=records["dcim.devicetype"])
        self.sync_items(items=records["dcim.platform"])
        self.sync_items(items=records["dcim.devicerole"])
        try:
            # This signal does not call for snapshot(), causing issue with branching plugin
            signals.post_save.disconnect(
                assign_virtualchassis_master, sender=VirtualChassis
            )
            self.sync_items(items=records["dcim.virtualchassis"])
            self.sync_devices(devices=records["dcim.device"], ingestion=ingestion)
            # The Device exists now, so we can update the master of the VC.
            # The logic is handled in transform maps.
            # Reset cached context on each VC record so get_transform_context
            # re-evaluates against the current DB state (master Device now exists).
            # Without this, the context cached during the 1st pass (master=None)
            # would be reused and VirtualChassis.master would never be assigned.
            for vc_record in records["dcim.virtualchassis"]:
                vc_record.reset_context()
            self.sync_items(items=records["dcim.virtualchassis"], stats=False)
        finally:
            signals.post_save.connect(
                assign_virtualchassis_master, sender=VirtualChassis
            )
        self.sync_items(items=records["dcim.interface"])
        self.sync_items(items=records["dcim.macaddress"])
        self.sync_items(items=records["dcim.inventoryitem"])
        self.sync_items(items=records["ipam.vlan"])
        self.sync_items(items=records["ipam.vrf"])
        self.sync_items(items=records["ipam.prefix"])
        self.sync_ip_addresses(ip_addresses=records["ipam.ipaddress"])

        # Make sure to clean queue (and memory) at the end
        self.events_clearer.clear()
