from __future__ import annotations

import asyncio
from datetime import datetime, timedelta, timezone

from typing import Any, Dict, List, Optional, Set

from fastapi import APIRouter, Depends, Header, HTTPException, Query
from fastapi.responses import JSONResponse
from sqlalchemy.ext.asyncio import AsyncSession

from .deps import idem_store, require_api_key
from ...db import get_session
from ...services.salesforce_gateway import (
    SalesforceAuthError,
    SalesforceConnectionError,
    SalesforceFieldAccessError,
    SalesforceGateway,
    SalesforceRateLimitError,
)
from ...integrations.salesforce import (
    BulkJobAbortedError,
    BulkJobFailedError,
    BulkResultFetchError,
)
from ...services import sfdc_import_idempotency as sfdc_idem
from ... import settings
from fmatch.core.b2c_bloom_filter import is_b2c_domain

import csv
import io
import math
import re
import time
import uuid
import logging
import json

log = logging.getLogger(__name__)

# US State abbreviation to full name mapping for Salesforce State/Country Picklists
_US_STATE_ABBREV_TO_FULL = {
    "AL": "Alabama",
    "AK": "Alaska",
    "AZ": "Arizona",
    "AR": "Arkansas",
    "CA": "California",
    "CO": "Colorado",
    "CT": "Connecticut",
    "DE": "Delaware",
    "FL": "Florida",
    "GA": "Georgia",
    "HI": "Hawaii",
    "ID": "Idaho",
    "IL": "Illinois",
    "IN": "Indiana",
    "IA": "Iowa",
    "KS": "Kansas",
    "KY": "Kentucky",
    "LA": "Louisiana",
    "ME": "Maine",
    "MD": "Maryland",
    "MA": "Massachusetts",
    "MI": "Michigan",
    "MN": "Minnesota",
    "MS": "Mississippi",
    "MO": "Missouri",
    "MT": "Montana",
    "NE": "Nebraska",
    "NV": "Nevada",
    "NH": "New Hampshire",
    "NJ": "New Jersey",
    "NM": "New Mexico",
    "NY": "New York",
    "NC": "North Carolina",
    "ND": "North Dakota",
    "OH": "Ohio",
    "OK": "Oklahoma",
    "OR": "Oregon",
    "PA": "Pennsylvania",
    "RI": "Rhode Island",
    "SC": "South Carolina",
    "SD": "South Dakota",
    "TN": "Tennessee",
    "TX": "Texas",
    "UT": "Utah",
    "VT": "Vermont",
    "VA": "Virginia",
    "WA": "Washington",
    "WV": "West Virginia",
    "WI": "Wisconsin",
    "WY": "Wyoming",
    "DC": "District of Columbia",
    "PR": "Puerto Rico",
    "VI": "Virgin Islands",
    "GU": "Guam",
    "AS": "American Samoa",
    "MP": "Northern Mariana Islands",
}

_US_STATE_FULL_TO_ABBREV = {
    name.upper(): code for code, name in _US_STATE_ABBREV_TO_FULL.items()
}

_CA_PROVINCE_ABBREV_TO_FULL = {
    "AB": "Alberta",
    "BC": "British Columbia",
    "MB": "Manitoba",
    "NB": "New Brunswick",
    "NL": "Newfoundland and Labrador",
    "NS": "Nova Scotia",
    "ON": "Ontario",
    "PE": "Prince Edward Island",
    "QC": "Quebec",
    "SK": "Saskatchewan",
    "NT": "Northwest Territories",
    "NU": "Nunavut",
    "YT": "Yukon",
}

_CA_PROVINCE_FULL_TO_ABBREV = {
    name.upper(): code for code, name in _CA_PROVINCE_ABBREV_TO_FULL.items()
}

_AU_STATE_ABBREV_TO_FULL = {
    "NSW": "New South Wales",
    "VIC": "Victoria",
    "QLD": "Queensland",
    "SA": "South Australia",
    "WA": "Western Australia",
    "TAS": "Tasmania",
    "ACT": "Australian Capital Territory",
    "NT": "Northern Territory",
}

_AU_STATE_FULL_TO_ABBREV = {
    name.upper(): code for code, name in _AU_STATE_ABBREV_TO_FULL.items()
}

_STATE_MAPS = {
    "US": (_US_STATE_ABBREV_TO_FULL, _US_STATE_FULL_TO_ABBREV),
    "CA": (_CA_PROVINCE_ABBREV_TO_FULL, _CA_PROVINCE_FULL_TO_ABBREV),
    "AU": (_AU_STATE_ABBREV_TO_FULL, _AU_STATE_FULL_TO_ABBREV),
}

_COUNTRY_CODE_TO_NAME = {
    "US": "United States",
    "CA": "Canada",
    "MX": "Mexico",
    "HK": "Hong Kong",
}

_COUNTRY_ALIAS_TO_CODE = {
    "USA": "US",
    "UNITED STATES": "US",
    "UNITED STATES OF AMERICA": "US",
    "U.S.": "US",
    "U.S.A": "US",
    "CANADA": "CA",
    "CAN": "CA",
    "MEXICO": "MX",
    "HONG KONG": "HK",
    "HONGKONG": "HK",
    "H.K.": "HK",
    "HK": "HK",
}

router = APIRouter(prefix="/api/v1/salesforce", tags=["salesforce"])

_PUSH_JOBS: Dict[str, Dict[str, Any]] = {}

_SOBJECT_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
_FIELD_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_\\.]*$")
_CAMPAIGN_ID_FIELD_NAMES = {"campaignid", "campaign_id", "campaignids"}
_CAMPAIGN_STATUS_FIELD_NAMES = {"campaignmemberstatus", "campaign_status"}
_SF_ID_RE = re.compile(r"^[A-Za-z0-9]{15}([A-Za-z0-9]{3})?$")
_CAMPAIGN_MEMBER_STATUS_CACHE: Dict[str, Dict[str, Any]] = {}
_CAMPAIGN_MEMBER_STATUS_CACHE_TTL = 3600
CAMPAIGN_VALIDATION_FAIL_CLOSED = "fail_closed"
CAMPAIGN_VALIDATION_BEST_EFFORT = "best_effort"
CAMPAIGN_MEMBER_BEHAVIOR_SKIP = "skip"
CAMPAIGN_MEMBER_BEHAVIOR_ERROR = "error"
COMPOSITE_INSERT_BATCH_SIZE = 25
CAMPAIGN_MEMBER_BATCH_SIZE = 25
COMPOSITE_CALL_WARNING_THRESHOLD_DEFAULT = 150
COMPOSITE_CALL_WARNING_THRESHOLD_ENTERPRISE = 500
COMPOSITE_CALL_HIGH_THRESHOLD = 1000
COMPOSITE_RATE_LIMIT_BACKOFF_SECONDS = 5

BULK_API_THRESHOLD = 500


class BulkMappingError(Exception):
    def __init__(
        self,
        message: str,
        detail: Optional[str] = None,
        issues: Optional[List[Dict[str, Any]]] = None,
    ) -> None:
        super().__init__(message)
        self.message = message
        self.detail = detail
        self.issues = issues or []


def _has_value(value: Any) -> bool:
    if value is None:
        return False
    if isinstance(value, str):
        return value.strip() != ""
    if isinstance(value, (list, tuple, set)):
        return len(value) > 0
    return True


def _normalize_mapping_value(value: Any) -> Optional[str]:
    if value is None:
        return None
    text = str(value)
    if isinstance(value, str):
        text = value.strip()
    return text if text != "" else None


def _get_row_value_case_insensitive(row: Dict[str, Any], key: str) -> Any:
    if key in row:
        return row.get(key)
    lower_key = key.lower()
    for candidate, value in row.items():
        if candidate.lower() == lower_key:
            return value
    return None


def _can_map_bulk_results_deterministically(
    operation: str,
    records: List[Dict[str, Any]],
    external_id_field: Optional[str],
) -> tuple[bool, Optional[str]]:
    op = (operation or "").lower()
    if op == "update":
        return True, "Id"
    if op == "upsert":
        return bool(external_id_field), external_id_field or None
    if op == "insert":
        return False, None
    return False, None


def _mapping_key_from_method(
    mapping_method: Optional[str], external_id_field: Optional[str]
) -> Optional[str]:
    if mapping_method == "bulk_by_id":
        return "Id"
    if mapping_method == "bulk_by_external_id":
        return external_id_field
    return None


def _execution_mode_from_mapping(
    mapping_method: Optional[str], *, dry_run: bool
) -> str:
    if dry_run:
        return "rest"
    if mapping_method and mapping_method.startswith("bulk"):
        return "bulk"
    return "rest"


def _normalize_country_field(value: Any) -> tuple[Optional[str], Optional[str]]:
    if not isinstance(value, str):
        return None, None

    raw = value.strip()
    if not raw:
        return None, None

    upper = raw.upper()
    code: Optional[str] = None
    label: Optional[str] = None

    if upper in _COUNTRY_CODE_TO_NAME:
        code = upper
        label = _COUNTRY_CODE_TO_NAME[upper]
    elif upper in _COUNTRY_ALIAS_TO_CODE:
        code = _COUNTRY_ALIAS_TO_CODE[upper]
        label = _COUNTRY_CODE_TO_NAME.get(code, raw.title())
    else:
        lower = raw.lower()
        for candidate_code, candidate_label in _COUNTRY_CODE_TO_NAME.items():
            if lower == candidate_label.lower():
                code = candidate_code
                label = candidate_label
                break

    if label is None:
        label = raw
    return label, code


def _normalize_state_field(
    value: Any, country_code: Optional[str]
) -> tuple[Optional[str], Optional[str]]:
    if not isinstance(value, str):
        return None, None

    raw = value.strip()
    if not raw:
        return None, None

    upper = raw.upper()
    candidates: List[str] = []
    if country_code:
        country_upper = country_code.upper()
        if country_upper in _STATE_MAPS:
            candidates.append(country_upper)
        elif country_upper == "USA":
            candidates.append("US")
    if not candidates:
        # Default assumption for legacy sheets that omit CountryCode
        candidates.append("US")

    for candidate in candidates:
        abbrev_map, full_map = _STATE_MAPS[candidate]
        if upper in abbrev_map:
            code = upper
            return abbrev_map[code], code

        code = full_map.get(upper)
        if code:
            return abbrev_map[code], code

    return raw, None


def _normalize_salesforce_data(
    sobject: str,
    data: Dict[str, Any],
    supported_fields: Optional[Set[str]] = None,
) -> Dict[str, Any]:
    """
    Normalize data before sending to Salesforce to handle:
    1. State/Country Picklists: Ensure label/code values align with Salesforce expectations
    2. Required fields: Provide defaults for missing required fields
    """
    normalized = dict(data)
    input_keys = set(data.keys())
    supported = set(supported_fields or [])

    def _can_set_generated_code_field(field_name: str) -> bool:
        # Keep caller-provided code fields untouched, but only auto-add code
        # variants when describe metadata confirms the field exists.
        if field_name in input_keys:
            return True
        if not supported:
            return True
        return field_name in supported

    # Normalize country fields (Country/CountryCode variants) first so downstream state logic can rely on codes.
    country_fields = [
        ("Country", "CountryCode"),
        ("MailingCountry", "MailingCountryCode"),
        ("BillingCountry", "BillingCountryCode"),
        ("ShippingCountry", "ShippingCountryCode"),
    ]
    primary_country_code: Optional[str] = None

    for label_field, code_field in country_fields:
        label, code = _normalize_country_field(normalized.get(label_field))
        code_label, code_from_field = _normalize_country_field(
            normalized.get(code_field)
        )

        if code_from_field:
            code = code_from_field
            if not label:
                label = code_label

        if label:
            # Only update if the value actually changes to avoid unnecessary churn
            if normalized.get(label_field) != label:
                normalized[label_field] = label
                log.debug(f"[SF IMPORT] Normalized {label_field} to '{label}'")

        if code:
            code_upper = code.upper()
            if _can_set_generated_code_field(code_field) and normalized.get(
                code_field
            ) != code_upper:
                normalized[code_field] = code_upper
                log.debug(f"[SF IMPORT] Normalized {code_field} to '{code_upper}'")
            if not primary_country_code:
                primary_country_code = code_upper

    # Normalize state fields and ensure matching StateCode values are present
    state_fields = [
        ("State", "StateCode"),
        ("MailingState", "MailingStateCode"),
        ("BillingState", "BillingStateCode"),
        ("ShippingState", "ShippingStateCode"),
    ]

    for label_field, code_field in state_fields:
        label, code = _normalize_state_field(
            normalized.get(label_field), primary_country_code
        )
        code_label, code_from_field = _normalize_state_field(
            normalized.get(code_field), primary_country_code
        )

        if code_from_field:
            code = code_from_field
            if not label and code_label:
                label = code_label

        if label and normalized.get(label_field) != label:
            normalized[label_field] = label
            if code:
                log.debug(f"[SF IMPORT] Normalized {label_field} to '{label}' ({code})")
            else:
                log.debug(f"[SF IMPORT] Normalized {label_field} to '{label}'")

        if code:
            code_upper = code.upper()
            if _can_set_generated_code_field(code_field) and normalized.get(
                code_field
            ) != code_upper:
                normalized[code_field] = code_upper
                log.debug(f"[SF IMPORT] Normalized {code_field} to '{code_upper}'")

    # Handle Lead-specific required fields
    if sobject.lower() == "lead":
        # Company is required on Lead but not always available
        if not _has_value(normalized.get("Company")):
            # Try to derive from other fields
            if _has_value(normalized.get("Email")):
                # Extract domain from email as fallback
                email = str(normalized["Email"])
                if "@" in email:
                    domain = email.split("@")[1].lower()
                    # Use bloom filter to check if this is a personal/disposable domain (57K+ domains)
                    if not is_b2c_domain(domain):
                        company_token = domain.split(".")[0]
                        company_name = company_token.title()
                        normalized["Company"] = company_name
                        log.debug(
                            f"[SF IMPORT] Derived Company '{company_name}' from email domain '{domain}'"
                        )
                    else:
                        log.debug(
                            f"[SF IMPORT] Skipping Company derivation - '{domain}' is a personal/disposable domain"
                        )

            # If still no company, use a generic placeholder
            if not _has_value(normalized.get("Company")):
                normalized["Company"] = "Unknown"
                log.warning(
                    "[SF IMPORT] No Company found, using placeholder 'Unknown'"
                )

    return normalized


def _find_unsupported_fields(
    data: Dict[str, Any], supported_fields: Optional[Set[str]]
) -> List[str]:
    if not supported_fields:
        return []
    return sorted(
        field for field in data.keys() if field and field not in supported_fields
    )


def _append_unsupported_field_issues(
    *,
    issues: List[Dict[str, Any]],
    row: int,
    sobject: str,
    fields: List[str],
    record_data: Dict[str, Any],
) -> None:
    for field in fields:
        issues.append(
            {
                "row": row,
                "field": field,
                "value": record_data.get(field),
                "code": "UNSUPPORTED_FIELD",
                "message": f"{field} is not available on {sobject} for this org.",
            }
        )


def _parse_salesforce_error(exc: Exception) -> tuple[str, str, Optional[str]]:
    """Extract message, code, and field from a Salesforce HTTP error."""
    message = str(exc)
    code = "API_ERROR"
    field = None

    response = getattr(exc, "response", None)
    if response is not None:
        content = getattr(response, "_content", None)
        if content:
            try:
                data = json.loads(content)
                if isinstance(data, list) and data:
                    sf_error = data[0]
                    message = sf_error.get("message", message)
                    code = sf_error.get("errorCode", code)
                    fields = sf_error.get("fields", [])
                    if isinstance(fields, list) and fields:
                        field = str(fields[0])
                    elif fields:
                        field = str(fields)
            except Exception:
                pass
    return message, code, field


def _parse_campaign_ids(value: Any) -> List[str]:
    """Parse campaign identifiers into a list."""
    if value is None:
        return []
    if isinstance(value, (list, tuple, set)):
        raw_values = value
    else:
        raw_values = re.split(r"[;,]", str(value))
    parsed: List[str] = []
    for item in raw_values:
        cid = str(item or "").strip()
        if cid:
            parsed.append(cid)
    return parsed


def _extract_campaign_metadata(data: Dict[str, Any]) -> Dict[str, Any]:
    """Remove campaign-related fields from record data and return metadata."""
    campaign_ids: List[str] = []
    campaign_status: Optional[str] = None

    for key in list(data.keys()):
        key_lower = key.lower()
        if key_lower in _CAMPAIGN_ID_FIELD_NAMES or key_lower.startswith("campaignid"):
            campaign_ids.extend(_parse_campaign_ids(data.pop(key)))
        elif key_lower in _CAMPAIGN_STATUS_FIELD_NAMES:
            raw_status = str(data.pop(key) or "").strip()
            if raw_status:
                campaign_status = raw_status

    # Ensure unique while preserving order
    seen = set()
    deduped_ids: List[str] = []
    for cid in campaign_ids:
        if cid not in seen:
            seen.add(cid)
            deduped_ids.append(cid)

    return {"campaign_ids": deduped_ids, "status": campaign_status}


def _record_supports_campaign(sobject: str) -> bool:
    return sobject.lower() in {"lead", "contact"}


def _normalize_campaign_validation_mode(value: Optional[str]) -> str:
    if not value:
        return CAMPAIGN_VALIDATION_FAIL_CLOSED
    mode = str(value).strip().lower()
    mode = mode.replace("-", "_")
    if mode == CAMPAIGN_VALIDATION_BEST_EFFORT:
        return CAMPAIGN_VALIDATION_BEST_EFFORT
    return CAMPAIGN_VALIDATION_FAIL_CLOSED


def _normalize_campaign_member_behavior(value: Optional[str]) -> str:
    if not value:
        return CAMPAIGN_MEMBER_BEHAVIOR_SKIP
    mode = str(value).strip().lower()
    if mode in {"error", "fail", "strict"}:
        return CAMPAIGN_MEMBER_BEHAVIOR_ERROR
    return CAMPAIGN_MEMBER_BEHAVIOR_SKIP


def _chunked(items: List[Any], size: int) -> List[List[Any]]:
    return [items[i : i + size] for i in range(0, len(items), size)]


def _estimate_campaign_memberships(records: List[Dict[str, Any]]) -> int:
    total = 0
    for record in records:
        if not isinstance(record, dict):
            continue
        record_data = {k: v for k, v in record.items() if k != "attributes"}
        campaign_meta = _extract_campaign_metadata(record_data)
        total += len(campaign_meta.get("campaign_ids") or [])
    return total


def _composite_warning_threshold(tier: Optional[str]) -> int:
    tier_norm = (tier or "free").upper()
    if tier_norm in {"ENT", "ENTERPRISE"}:
        return COMPOSITE_CALL_WARNING_THRESHOLD_ENTERPRISE
    return COMPOSITE_CALL_WARNING_THRESHOLD_DEFAULT


def _compute_import_warnings(
    records: List[Dict[str, Any]],
    *,
    use_composite_insert: bool,
    add_to_campaign: bool,
    tier: Optional[str],
    dry_run: bool,
) -> List[Dict[str, Any]]:
    projected_calls = 0
    detail: Dict[str, Any] = {}

    if use_composite_insert:
        insert_calls = int(math.ceil(len(records) / COMPOSITE_INSERT_BATCH_SIZE))
        projected_calls += insert_calls
        detail["keyless_insert_calls"] = insert_calls

    if add_to_campaign:
        membership_count = _estimate_campaign_memberships(records)
        if membership_count:
            campaign_calls = int(
                math.ceil(membership_count / CAMPAIGN_MEMBER_BATCH_SIZE)
            )
            projected_calls += campaign_calls
            detail["campaign_member_calls"] = campaign_calls
            detail["campaign_memberships"] = membership_count

    warnings: List[Dict[str, Any]] = []
    threshold = _composite_warning_threshold(tier)
    if projected_calls >= threshold:
        severity = (
            "high"
            if projected_calls >= COMPOSITE_CALL_HIGH_THRESHOLD
            else "warning"
        )
        phase = "dry_run" if dry_run else "execute"
        warnings.append(
            {
                "code": "API_LIMIT_RISK",
                "severity": severity,
                "phase": phase,
                "message": (
                    "Import may generate a high number of Composite API calls. "
                    "Consider reducing batch size or providing a unique key for bulk."
                ),
                "projected_calls": projected_calls,
                "threshold": threshold,
                "details": detail,
            }
        )
    return warnings


def _attach_warnings(
    response_body: Dict[str, Any], warnings: List[Dict[str, Any]]
) -> None:
    if warnings:
        response_body["warnings"] = warnings


def _derive_import_status(issues: List[Dict[str, Any]]) -> str:
    codes = {issue.get("code") for issue in issues}
    if (
        "CAMPAIGN_VALIDATION_QUERY_FAILED" in codes
        or "CAMPAIGN_STATUS_QUERY_FAILED" in codes
    ):
        return "validation_failed"
    return "completed" if not issues else "partial_success"


async def validate_campaign_ids(
    gateway: SalesforceGateway,
    campaign_ids: set[str],
    *,
    validation_mode: str = CAMPAIGN_VALIDATION_FAIL_CLOSED,
) -> tuple[set[str], List[Dict[str, Any]], bool]:
    """Validate that campaign IDs exist and are accessible."""
    if not campaign_ids:
        return set(), [], False

    valid_ids: set[str] = set()
    issues: List[Dict[str, Any]] = []
    validation_failed = False
    query_failed = False
    candidate_ids: List[str] = []
    for cid in sorted(campaign_ids):
        if not _SF_ID_RE.fullmatch(cid):
            issues.append(
                {
                    "campaign_id": cid,
                    "code": "CAMPAIGN_ID_INVALID",
                    "message": f"Campaign ID '{cid}' is not a valid Salesforce ID.",
                }
            )
            continue
        candidate_ids.append(cid)

    if not candidate_ids:
        return set(), issues, False

    for batch in _chunked(candidate_ids, 200):
        ids_str = "','".join(batch)
        query = (
            "SELECT Id, Name, IsActive FROM Campaign "
            f"WHERE Id IN ('{ids_str}')"
        )
        try:
            results = await gateway.soql(query)
            records = results.get("records") or []
            for record in records:
                record_id = record.get("Id")
                if not record_id:
                    continue
                if record.get("IsActive", True):
                    valid_ids.add(record_id)
                else:
                    issues.append(
                        {
                            "campaign_id": record_id,
                            "campaign_name": record.get("Name"),
                            "code": "CAMPAIGN_INACTIVE",
                            "message": (
                                f"Campaign '{record.get('Name')}' is inactive."
                            ),
                        }
                    )
        except Exception as exc:  # noqa: BLE001
            log.warning("[CAMPAIGN] Failed to validate campaign batch: %s", exc)
            if validation_mode == CAMPAIGN_VALIDATION_BEST_EFFORT:
                valid_ids.update(batch)
            else:
                validation_failed = True
                if not query_failed:
                    issues.append(
                        {
                            "campaign_id": None,
                            "code": "CAMPAIGN_VALIDATION_QUERY_FAILED",
                            "message": (
                                "Failed to validate campaign IDs due to a Salesforce query error."
                            ),
                            "detail": str(exc),
                        }
                    )
                    query_failed = True

    if validation_failed and validation_mode != CAMPAIGN_VALIDATION_BEST_EFFORT:
        return valid_ids, issues, True

    invalid_ids = {issue["campaign_id"] for issue in issues if issue.get("campaign_id")}
    missing = set(candidate_ids) - valid_ids - invalid_ids
    for cid in missing:
        issues.append(
            {
                "campaign_id": cid,
                "code": "CAMPAIGN_NOT_FOUND",
                "message": f"Campaign {cid} not found or not accessible.",
            }
        )

    return valid_ids, issues, False


async def validate_campaign_member_statuses(
    gateway: SalesforceGateway,
    campaign_ids: set[str],
    *,
    validation_mode: str = CAMPAIGN_VALIDATION_FAIL_CLOSED,
) -> tuple[Dict[str, Dict[str, str]], List[Dict[str, Any]], bool]:
    """Validate CampaignMember statuses per campaign."""
    if not campaign_ids:
        return {}, [], False

    status_map: Dict[str, Dict[str, str]] = {}
    issues: List[Dict[str, Any]] = []
    validation_failed = False
    query_failed = False

    for batch in _chunked(sorted(campaign_ids), 200):
        ids_str = "','".join(batch)
        query = (
            "SELECT CampaignId, Label FROM CampaignMemberStatus "
            f"WHERE CampaignId IN ('{ids_str}')"
        )
        try:
            results = await gateway.soql(query)
            records = results.get("records") or []
            for record in records:
                campaign_id = record.get("CampaignId")
                label = record.get("Label")
                if not campaign_id or not label:
                    continue
                status_map.setdefault(campaign_id, {})[label.lower()] = label
        except Exception as exc:  # noqa: BLE001
            log.warning(
                "[CAMPAIGN] Failed to fetch CampaignMemberStatus for batch: %s", exc
            )
            if validation_mode == CAMPAIGN_VALIDATION_BEST_EFFORT:
                fallback = await get_valid_campaign_member_statuses(gateway)
                fallback_map = {value.lower(): value for value in fallback}
                for cid in batch:
                    status_map.setdefault(cid, dict(fallback_map))
            else:
                validation_failed = True
                if not query_failed:
                    issues.append(
                        {
                            "campaign_id": None,
                            "code": "CAMPAIGN_STATUS_QUERY_FAILED",
                            "message": (
                                "Failed to validate campaign member statuses due to a Salesforce query error."
                            ),
                            "detail": str(exc),
                        }
                    )
                    query_failed = True

    if validation_failed and validation_mode != CAMPAIGN_VALIDATION_BEST_EFFORT:
        return status_map, issues, True

    missing = set(campaign_ids) - set(status_map.keys())
    for cid in missing:
        issues.append(
            {
                "campaign_id": cid,
                "code": "CAMPAIGN_STATUS_NOT_FOUND",
                "message": f"Campaign {cid} has no available member statuses.",
            }
        )

    return status_map, issues, False


async def get_valid_campaign_member_statuses(
    gateway: SalesforceGateway,
) -> set[str]:
    """Get valid CampaignMember.Status picklist values."""
    cache_key = gateway.org_id or gateway.tenant_id
    now = time.time()
    cached = _CAMPAIGN_MEMBER_STATUS_CACHE.get(cache_key)
    if cached and now - cached["ts"] < _CAMPAIGN_MEMBER_STATUS_CACHE_TTL:
        return set(cached["values"])

    try:
        describe = await gateway.describe("CampaignMember")
        status_field = next(
            (f for f in describe.get("fields", []) if f.get("name") == "Status"),
            None,
        )
        if status_field:
            valid = {
                pv["value"]
                for pv in status_field.get("picklistValues", [])
                if pv.get("active", True)
            }
            _CAMPAIGN_MEMBER_STATUS_CACHE[cache_key] = {
                "ts": now,
                "values": set(valid),
            }
            return valid
    except Exception as exc:  # noqa: BLE001
        log.warning(
            "[CAMPAIGN] Failed to fetch CampaignMember status picklist: %s", exc
        )

    return {"Sent", "Responded"}


async def validate_campaign_status(
    gateway: SalesforceGateway,
    status: str,
) -> tuple[bool, Optional[str]]:
    """Validate campaign member status value."""
    if not status:
        return False, None
    valid_statuses = await get_valid_campaign_member_statuses(gateway)
    if status in valid_statuses:
        return True, status
    status_lower = status.lower()
    for valid in valid_statuses:
        if valid.lower() == status_lower:
            return True, valid
    return False, None


async def create_campaign_members_batched(
    gateway: SalesforceGateway,
    assignments: List[Dict[str, Any]],
    *,
    default_status: str = "Responded",
    duplicate_behavior: str = CAMPAIGN_MEMBER_BEHAVIOR_SKIP,
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
    """Create CampaignMember records via Composite API batches."""
    successes: List[Dict[str, Any]] = []
    failures: List[Dict[str, Any]] = []
    duplicate_behavior = _normalize_campaign_member_behavior(duplicate_behavior)

    payloads: List[Dict[str, Any]] = []
    for assignment in assignments:
        record_id = assignment["record_id"]
        record_type = assignment["record_type"]
        campaign_id = assignment["campaign_id"]
        status = assignment.get("status") or default_status
        member_field = "LeadId" if record_type == "Lead" else "ContactId"
        payloads.append(
            {
                "assignment": assignment,
                "payload": {
                    "CampaignId": campaign_id,
                    member_field: record_id,
                    "Status": status,
                },
            }
        )

    for batch in _chunked(payloads, CAMPAIGN_MEMBER_BATCH_SIZE):
        composite_request = {
            "allOrNone": False,
            "compositeRequest": [
                {
                    "method": "POST",
                    "url": (
                        f"/services/data/{settings.SF_API_VERSION}"
                        "/sobjects/CampaignMember"
                    ),
                    "referenceId": f"cm_{idx}",
                    "body": item["payload"],
                }
                for idx, item in enumerate(batch)
            ],
        }

        try:
            response = None
            for attempt in range(2):
                try:
                    response = await gateway.composite(composite_request)
                    break
                except SalesforceRateLimitError as exc:
                    delay = (
                        exc.retry_after or COMPOSITE_RATE_LIMIT_BACKOFF_SECONDS
                    )
                    log.warning(
                        "[CAMPAIGN] Composite rate limited; retrying in %ss",
                        int(math.ceil(delay)),
                    )
                    await asyncio.sleep(delay)
                    if attempt == 1:
                        raise

            composite_response = (
                response.get("compositeResponse") if response else None
            )
            if not isinstance(composite_response, list):
                raise RuntimeError("Composite response missing results")

            rate_limited = False
            for idx, result in enumerate(composite_response):
                assignment = batch[idx]["assignment"]
                http_status = int(result.get("httpStatusCode") or 0)
                body = result.get("body") or {}

                if http_status in (200, 201):
                    successes.append(
                        {
                            "row": assignment.get("row"),
                            "record_id": assignment["record_id"],
                            "campaign_id": assignment["campaign_id"],
                            "campaign_member_id": body.get("id"),
                            "status": "created",
                        }
                    )
                    continue

                errors = body if isinstance(body, list) else body.get("errors") or []
                error_code = None
                error_message = None
                if errors and isinstance(errors, list):
                    first = errors[0] if isinstance(errors[0], dict) else {}
                    if isinstance(first, dict):
                        error_code = first.get("errorCode")
                        error_message = first.get("message")
                if not error_message:
                    error_message = (
                        body.get("message") if isinstance(body, dict) else str(body)
                    )

                if (
                    error_code == "REQUEST_LIMIT_EXCEEDED"
                    or "REQUEST_LIMIT_EXCEEDED" in str(errors).upper()
                ):
                    rate_limited = True

                if error_code == "DUPLICATE_VALUE" or "DUPLICATE" in str(body).upper():
                    if duplicate_behavior == CAMPAIGN_MEMBER_BEHAVIOR_ERROR:
                        failures.append(
                            {
                                "row": assignment.get("row"),
                                "record_id": assignment["record_id"],
                                "campaign_id": assignment["campaign_id"],
                                "code": error_code or "DUPLICATE_VALUE",
                                "message": error_message
                                or "Campaign member already exists.",
                            }
                        )
                        continue
                    successes.append(
                        {
                            "row": assignment.get("row"),
                            "record_id": assignment["record_id"],
                            "campaign_id": assignment["campaign_id"],
                            "campaign_member_id": None,
                            "status": "already_member",
                        }
                    )
                    continue

                failures.append(
                    {
                        "row": assignment.get("row"),
                        "record_id": assignment["record_id"],
                        "campaign_id": assignment["campaign_id"],
                        "code": error_code or "CAMPAIGN_MEMBER_ERROR",
                        "message": error_message or "Campaign member creation failed.",
                    }
                )

            if rate_limited:
                await asyncio.sleep(COMPOSITE_RATE_LIMIT_BACKOFF_SECONDS)
        except Exception as exc:  # noqa: BLE001
            for item in batch:
                assignment = item["assignment"]
                failures.append(
                    {
                        "row": assignment.get("row"),
                        "record_id": assignment["record_id"],
                        "campaign_id": assignment["campaign_id"],
                        "code": "COMPOSITE_ERROR",
                        "message": str(exc),
                    }
                )

    return successes, failures


async def _process_campaign_memberships(
    gateway: SalesforceGateway,
    *,
    sobject: str,
    assignments: List[Dict[str, Any]],
    default_status: Optional[str],
    results: List[Dict[str, Any]],
    issues: List[Dict[str, Any]],
    validation_mode: str = CAMPAIGN_VALIDATION_FAIL_CLOSED,
    duplicate_behavior: str = CAMPAIGN_MEMBER_BEHAVIOR_SKIP,
) -> None:
    """
    Create CampaignMember records for successful imports.

    assignments: [{"row": idx, "record_id": "...", "campaign_ids": [...], "status": "..."}]
    """
    if not assignments or not _record_supports_campaign(sobject):
        return

    validation_mode = _normalize_campaign_validation_mode(validation_mode)
    default_status = (default_status or "Responded").strip() or None

    all_campaign_ids: set[str] = set()
    for assignment in assignments:
        all_campaign_ids.update(assignment.get("campaign_ids") or [])

    valid_campaign_ids, campaign_issues, campaign_validation_failed = (
        await validate_campaign_ids(
            gateway, all_campaign_ids, validation_mode=validation_mode
        )
    )
    campaign_issue_by_id = {
        issue["campaign_id"]: issue
        for issue in campaign_issues
        if issue.get("campaign_id")
    }
    for issue in campaign_issues:
        if issue.get("code") == "CAMPAIGN_VALIDATION_QUERY_FAILED":
            issues.append(
                {
                    "row": -1,
                    "field": "CampaignId",
                    "code": issue.get("code"),
                    "message": issue.get("message"),
                    "detail": issue.get("detail"),
                }
            )
            break

    if campaign_validation_failed and validation_mode == CAMPAIGN_VALIDATION_FAIL_CLOSED:
        for assignment in assignments:
            row_index = assignment.get("row")
            campaign_ids = assignment.get("campaign_ids") or []
            for campaign_id in campaign_ids:
                message = (
                    "Campaign validation failed; campaign membership not created."
                )
                issues.append(
                    {
                        "row": row_index,
                        "field": "CampaignId",
                        "code": "CAMPAIGN_VALIDATION_QUERY_FAILED",
                        "message": message,
                        "value": campaign_id,
                    }
                )
                if row_index is not None and 0 <= row_index < len(results):
                    results[row_index].setdefault("campaignErrors", []).append(
                        f"{campaign_id}: {message}"
                    )
        return

    status_map, status_issues, status_validation_failed = (
        await validate_campaign_member_statuses(
            gateway, valid_campaign_ids, validation_mode=validation_mode
        )
    )
    status_issue_by_id = {
        issue["campaign_id"]: issue
        for issue in status_issues
        if issue.get("campaign_id")
    }
    for issue in status_issues:
        if issue.get("code") == "CAMPAIGN_STATUS_QUERY_FAILED":
            issues.append(
                {
                    "row": -1,
                    "field": "CampaignMemberStatus",
                    "code": issue.get("code"),
                    "message": issue.get("message"),
                    "detail": issue.get("detail"),
                }
            )
            break

    if (
        status_validation_failed
        and validation_mode == CAMPAIGN_VALIDATION_FAIL_CLOSED
    ):
        for assignment in assignments:
            row_index = assignment.get("row")
            campaign_ids = assignment.get("campaign_ids") or []
            for campaign_id in campaign_ids:
                message = (
                    "Campaign status validation failed; campaign membership not created."
                )
                issues.append(
                    {
                        "row": row_index,
                        "field": "CampaignMemberStatus",
                        "code": "CAMPAIGN_STATUS_QUERY_FAILED",
                        "message": message,
                        "value": campaign_id,
                    }
                )
                if row_index is not None and 0 <= row_index < len(results):
                    results[row_index].setdefault("campaignErrors", []).append(
                        f"{campaign_id}: {message}"
                    )
        return

    flat_assignments: List[Dict[str, Any]] = []
    record_type = "Lead" if sobject.lower() == "lead" else "Contact"

    for assignment in assignments:
        row_index = assignment.get("row")
        record_id = assignment.get("record_id")
        campaign_ids = assignment.get("campaign_ids") or []
        if not record_id or not campaign_ids:
            continue

        row_status = assignment.get("status")
        if row_status is not None:
            row_status = str(row_status).strip()
            if row_status == "":
                row_status = None
        if not row_status:
            row_status = default_status

        for campaign_id in campaign_ids:
            if campaign_id not in valid_campaign_ids:
                issue = campaign_issue_by_id.get(campaign_id)
                message = (
                    issue.get("message")
                    if issue
                    else f"Campaign {campaign_id} not found or not accessible."
                )
                code = issue.get("code") if issue else "CAMPAIGN_NOT_FOUND"
                issues.append(
                    {
                        "row": row_index,
                        "field": "CampaignId",
                        "code": code,
                        "message": message,
                        "value": campaign_id,
                    }
                )
                if row_index is not None and 0 <= row_index < len(results):
                    results[row_index].setdefault("campaignErrors", []).append(
                        f"{campaign_id}: {message}"
                    )
                continue

            status_candidates = status_map.get(campaign_id) or {}
            if not status_candidates:
                issue = status_issue_by_id.get(campaign_id)
                message = (
                    issue.get("message")
                    if issue
                    else (
                        f"Campaign member statuses for {campaign_id} could not be validated."
                    )
                )
                code = issue.get("code") if issue else "CAMPAIGN_STATUS_NOT_FOUND"
                issues.append(
                    {
                        "row": row_index,
                        "field": "CampaignMemberStatus",
                        "code": code,
                        "message": message,
                        "value": campaign_id,
                    }
                )
                if row_index is not None and 0 <= row_index < len(results):
                    results[row_index].setdefault("campaignErrors", []).append(
                        f"{campaign_id}: {message}"
                    )
                continue

            if not row_status:
                message = "Campaign member status is required."
                issues.append(
                    {
                        "row": row_index,
                        "field": "CampaignMemberStatus",
                        "code": "MISSING_STATUS",
                        "message": message,
                        "value": campaign_id,
                    }
                )
                if row_index is not None and 0 <= row_index < len(results):
                    results[row_index].setdefault("campaignErrors", []).append(
                        f"{campaign_id}: {message}"
                    )
                continue

            canonical_status = status_candidates.get(row_status.lower())
            if not canonical_status:
                valid_values = sorted(status_candidates.values())
                message = (
                    f"Campaign member status '{row_status}' is not valid for "
                    f"campaign {campaign_id}."
                )
                if valid_values:
                    message = (
                        f"{message} Valid statuses: {', '.join(valid_values)}."
                    )
                issues.append(
                    {
                        "row": row_index,
                        "field": "CampaignMemberStatus",
                        "code": "INVALID_STATUS",
                        "message": message,
                        "value": row_status,
                    }
                )
                if row_index is not None and 0 <= row_index < len(results):
                    results[row_index].setdefault("campaignErrors", []).append(
                        f"{campaign_id}: {message}"
                    )
                continue

            flat_assignments.append(
                {
                    "row": row_index,
                    "record_id": record_id,
                    "record_type": record_type,
                    "campaign_id": campaign_id,
                    "status": canonical_status,
                }
            )

    if not flat_assignments:
        return

    successes, failures = await create_campaign_members_batched(
        gateway,
        flat_assignments,
        default_status=default_status or "Responded",
        duplicate_behavior=duplicate_behavior,
    )

    for success in successes:
        row_index = success.get("row")
        campaign_id = success.get("campaign_id")
        if row_index is not None and 0 <= row_index < len(results):
            results[row_index].setdefault("campaignMemberships", []).append(
                campaign_id
            )

    for failure in failures:
        row_index = failure.get("row")
        campaign_id = failure.get("campaign_id")
        message = failure.get("message") or "Campaign member creation failed."
        issues.append(
            {
                "row": row_index,
                "field": "CampaignId",
                "code": failure.get("code") or "CAMPAIGN_MEMBER_ERROR",
                "message": message,
                "value": campaign_id,
            }
        )
        if row_index is not None and 0 <= row_index < len(results):
            results[row_index].setdefault("campaignErrors", []).append(
                f"{campaign_id}: {message}"
            )


async def _process_rest_import(
    records_raw: List[Dict[str, Any]],
    *,
    sobject: str,
    operation: str,
    external_id_field: Optional[str],
    gateway: SalesforceGateway,
    dry_run: bool,
    required_fields: List[str],
    supported_fields: Optional[Set[str]],
    add_to_campaign: bool,
    default_campaign_status: Optional[str],
    campaign_validation_mode: str = CAMPAIGN_VALIDATION_FAIL_CLOSED,
    campaign_member_behavior: str = CAMPAIGN_MEMBER_BEHAVIOR_SKIP,
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]:
    """Process import using standard REST API calls (existing behaviour)."""
    results: List[Dict[str, Any]] = []
    issues: List[Dict[str, Any]] = []
    success_count = 0
    total_records = len(records_raw)
    campaign_assignments: List[Dict[str, Any]] = []

    for idx, record in enumerate(records_raw):
        if not isinstance(record, dict):
            issues.append(
                {
                    "row": idx,
                    "code": "INVALID_RECORD",
                    "message": "Record must be an object.",
                }
            )
            results.append(
                {
                    "success": False,
                    "ok": False,
                    "status": "invalid",
                    "error": "Record must be an object.",
                    "recordId": None,
                    "id": None,
                }
            )
            continue

        record_data = {k: v for k, v in record.items() if k != "attributes"}
        record_id = record_data.pop("Id", None) or record_data.pop("id", None)

        if idx == 0:
            log.info(f"[SF IMPORT] First record keys: {list(record.keys())}")
            log.info(
                f"[SF IMPORT] First record_data keys after processing: {list(record_data.keys())}"
            )

        campaign_meta = _extract_campaign_metadata(record_data)

        if dry_run:
            log.debug(f"[SF IMPORT] DRY RUN: Validating record {idx}")
            unsupported_fields = _find_unsupported_fields(record_data, supported_fields)
            if unsupported_fields:
                _append_unsupported_field_issues(
                    issues=issues,
                    row=idx,
                    sobject=sobject,
                    fields=unsupported_fields,
                    record_data=record_data,
                )
                results.append(
                    {
                        "success": False,
                        "ok": False,
                        "status": "validation_failed",
                        "errors": [
                            f"{field} is not available on {sobject} for this org."
                            for field in unsupported_fields
                        ],
                        "recordId": record_id,
                        "id": record_id,
                    }
                )
                continue

            missing = [
                field
                for field in required_fields
                if not _has_value(record_data.get(field))
            ]
            if missing:
                for field in missing:
                    issues.append(
                        {
                            "row": idx,
                            "field": field,
                            "value": record_data.get(field),
                            "code": "REQUIRED_FIELD_MISSING",
                            "message": f"{field} is required.",
                        }
                    )
                results.append(
                    {
                        "success": False,
                        "ok": False,
                        "status": "validation_failed",
                        "errors": [f"{field} is required." for field in missing],
                        "recordId": record_id,
                        "id": record_id,
                    }
                )
                continue

            results.append(
                {
                    "success": True,
                    "ok": True,
                    "status": "validated",
                    "recordId": record_id,
                    "id": record_id,
                }
            )
            success_count += 1
            continue

        try:
            sf_response: Dict[str, Any]
            status_label = "created"
            out_id: Optional[str] = None

            log.debug(
                f"[SF IMPORT] Record {idx}: operation={operation}, record_id={record_id}, external_id_field={external_id_field}"
            )

            if operation == "update" or (
                record_id and operation in {"insert", "upsert"}
            ):
                log.info(f"[SF IMPORT] Taking UPDATE path for record {idx}")
                if not record_id:
                    issues.append(
                        {
                            "row": idx,
                            "code": "MISSING_ID",
                            "message": "Record is missing Id for update operation.",
                        }
                    )
                    results.append(
                        {
                            "success": False,
                            "ok": False,
                            "status": "missing_id",
                            "recordId": None,
                            "id": None,
                        }
                    )
                    continue
                normalized_data = _normalize_salesforce_data(
                    sobject, record_data, supported_fields
                )
                unsupported_fields = _find_unsupported_fields(
                    normalized_data, supported_fields
                )
                if unsupported_fields:
                    _append_unsupported_field_issues(
                        issues=issues,
                        row=idx,
                        sobject=sobject,
                        fields=unsupported_fields,
                        record_data=normalized_data,
                    )
                    results.append(
                        {
                            "success": False,
                            "ok": False,
                            "status": "unsupported_field",
                            "recordId": record_id,
                            "id": record_id,
                            "errors": [
                                f"{field} is not available on {sobject} for this org."
                                for field in unsupported_fields
                            ],
                            "error": (
                                f"Unsupported fields for {sobject}: "
                                + ", ".join(unsupported_fields)
                            ),
                        }
                    )
                    continue
                log.info(
                    f"[SF IMPORT] About to UPDATE existing record {record_id} for idx {idx}"
                )
                # Bypass gateway security restriction for import wizard - use client directly
                sf_response = await gateway.update(
                    sobject, record_id, normalized_data, enforce_lead_guard=False
                )
                log.info(
                    f"[SF IMPORT] UPDATE completed for idx {idx}, response: {sf_response}"
                )
                out_id = record_id
                status_label = "updated"
            elif operation == "upsert" and external_id_field:
                existing_id: Optional[str] = None
                ext_value = record_data.get(external_id_field)

                if not _has_value(ext_value):
                    log.warning(
                        f"[SF IMPORT] Skipping record {idx}: upsert requires {external_id_field} field, but value is missing"
                    )
                    issues.append(
                        {
                            "row": idx,
                            "field": external_id_field,
                            "value": ext_value,
                            "code": "MISSING_EXTERNAL_ID",
                            "message": f"Upsert operation requires {external_id_field} field but value is missing. Skipping record.",
                        }
                    )
                    results.append(
                        {
                            "success": False,
                            "ok": False,
                            "status": "skipped",
                            "recordId": None,
                            "id": None,
                            "error": f"Missing required field: {external_id_field}",
                        }
                    )
                    continue

                quoted_value = str(ext_value).replace("\\", "\\\\").replace("'", "\\'")
                query = f"SELECT Id FROM {sobject} WHERE {external_id_field} = '{quoted_value}' LIMIT 1"
                try:
                    log.debug(f"[SF IMPORT] SOQL lookup for record {idx}: {query}")
                    lookup = await gateway.soql(query, fls_guard=False)
                    log.critical(
                        f"[SF IMPORT] SOQL COMPLETED for record {idx}, got {len(lookup.get('records', []))} records"
                    )
                    records = lookup.get("records") or []
                    if records:
                        existing_id = records[0].get("Id") or records[0].get("id")
                        log.debug(f"[SF IMPORT] Found existing record: {existing_id}")
                except SalesforceFieldAccessError:
                    raise
                except Exception as exc:  # noqa: BLE001
                    log.warning(
                        "SOQL lookup failed for external id %s on %s: %s",
                        external_id_field,
                        sobject,
                        exc,
                    )

                log.critical(
                    f"[SF IMPORT] Decision point for record {idx}: existing_id={existing_id}"
                )
                normalized_data = _normalize_salesforce_data(
                    sobject, record_data, supported_fields
                )
                unsupported_fields = _find_unsupported_fields(
                    normalized_data, supported_fields
                )
                if unsupported_fields:
                    _append_unsupported_field_issues(
                        issues=issues,
                        row=idx,
                        sobject=sobject,
                        fields=unsupported_fields,
                        record_data=normalized_data,
                    )
                    results.append(
                        {
                            "success": False,
                            "ok": False,
                            "status": "unsupported_field",
                            "recordId": existing_id,
                            "id": existing_id,
                            "errors": [
                                f"{field} is not available on {sobject} for this org."
                                for field in unsupported_fields
                            ],
                            "error": (
                                f"Unsupported fields for {sobject}: "
                                + ", ".join(unsupported_fields)
                            ),
                        }
                    )
                    continue
                if existing_id:
                    log.info(
                        f"[SF IMPORT] About to UPDATE existing record {existing_id} for idx {idx}"
                    )
                    # Bypass gateway security restriction for import wizard - use client directly
                    sf_response = await gateway.update(
                        sobject, existing_id, normalized_data, enforce_lead_guard=False
                    )
                    log.info(
                        f"[SF IMPORT] UPDATE completed for idx {idx}, response: {sf_response}"
                    )
                    out_id = existing_id
                    status_label = "updated"
                else:
                    log.info(f"[SF IMPORT] About to CREATE new record for idx {idx}")
                    sf_response = await gateway.create(sobject, normalized_data)
                    log.info(
                        f"[SF IMPORT] CREATE completed for idx {idx}, response: {sf_response}"
                    )
                    out_id = sf_response.get("id")
                    status_label = "created"
            else:
                log.info(f"[SF IMPORT] Creating {sobject} record {idx}/{total_records}")
                normalized_data = _normalize_salesforce_data(
                    sobject, record_data, supported_fields
                )
                unsupported_fields = _find_unsupported_fields(
                    normalized_data, supported_fields
                )
                if unsupported_fields:
                    _append_unsupported_field_issues(
                        issues=issues,
                        row=idx,
                        sobject=sobject,
                        fields=unsupported_fields,
                        record_data=normalized_data,
                    )
                    results.append(
                        {
                            "success": False,
                            "ok": False,
                            "status": "unsupported_field",
                            "recordId": record_id,
                            "id": record_id,
                            "errors": [
                                f"{field} is not available on {sobject} for this org."
                                for field in unsupported_fields
                            ],
                            "error": (
                                f"Unsupported fields for {sobject}: "
                                + ", ".join(unsupported_fields)
                            ),
                        }
                    )
                    continue
                sf_response = await gateway.create(sobject, normalized_data)
                out_id = sf_response.get("id")
                status_label = "created"

            success = bool(sf_response.get("success", True))
            errors = sf_response.get("errors") or []
            if success:
                success_count += 1
                if add_to_campaign and campaign_meta["campaign_ids"] and out_id:
                    campaign_assignments.append(
                        {
                            "row": idx,
                            "record_id": out_id,
                            "campaign_ids": campaign_meta["campaign_ids"],
                            "status": campaign_meta["status"],
                        }
                    )
            else:
                if errors:
                    for err in errors:
                        if isinstance(err, dict):
                            issues.append(
                                {
                                    "row": idx,
                                    "field": (err.get("fields") or [None])[0],
                                    "value": None,
                                    "code": err.get("statusCode") or "API_ERROR",
                                    "message": err.get("message") or "Salesforce error",
                                }
                            )
                        else:
                            issues.append(
                                {
                                    "row": idx,
                                    "field": None,
                                    "value": None,
                                    "code": "API_ERROR",
                                    "message": str(err),
                                }
                            )
                else:
                    issues.append(
                        {
                            "row": idx,
                            "field": None,
                            "value": None,
                            "code": "API_ERROR",
                            "message": sf_response.get("message")
                            or "Salesforce returned an error.",
                        }
                    )

            result_entry = {
                "success": success,
                "ok": success,
                "status": status_label if success else "failed",
                "recordId": out_id,
                "id": out_id,
                "errors": errors,
                "error": None if success else sf_response.get("message"),
            }
            if add_to_campaign and campaign_meta["campaign_ids"]:
                result_entry["campaignRequested"] = campaign_meta["campaign_ids"]
            results.append(result_entry)

        except HTTPException:
            raise
        except SalesforceFieldAccessError as exc:
            message = str(exc)
            issues.append(
                {
                    "row": idx,
                    "code": "FIELD_ACCESS",
                    "message": message,
                }
            )
            results.append(
                {
                    "success": False,
                    "ok": False,
                    "status": "field_access_error",
                    "recordId": record_id,
                    "id": record_id,
                    "error": message,
                }
            )
        except SalesforceRateLimitError as exc:
            headers: Dict[str, str] = {}
            if exc.retry_after:
                headers["Retry-After"] = str(int(math.ceil(exc.retry_after)))
            raise HTTPException(
                status_code=429,
                detail={"error": "RATE_LIMIT", "message": str(exc)},
                headers=headers or None,
            )
        except SalesforceAuthError as exc:
            raise HTTPException(
                status_code=401,
                detail={"error": "AUTH_ERROR", "message": str(exc)},
            )
        except SalesforceConnectionError as exc:
            raise HTTPException(
                status_code=503,
                detail={"error": "SFDC_UNAVAILABLE", "message": str(exc)},
            )
        except Exception as exc:  # noqa: BLE001
            log.exception(
                "Unexpected Salesforce import error for account %s", gateway.tenant_id
            )
            message, code, field = _parse_salesforce_error(exc)
            issues.append(
                {
                    "row": idx,
                    "field": field,
                    "code": code,
                    "message": message,
                }
            )
            results.append(
                {
                    "success": False,
                    "ok": False,
                    "status": "exception",
                    "recordId": record_id,
                    "id": record_id,
                    "error": message,
                }
            )

    if add_to_campaign and campaign_assignments:
        await _process_campaign_memberships(
            gateway,
            sobject=sobject,
            assignments=campaign_assignments,
            default_status=default_campaign_status,
            results=results,
            issues=issues,
            validation_mode=campaign_validation_mode,
            duplicate_behavior=campaign_member_behavior,
        )

    return results, issues, success_count


async def _process_composite_insert(
    records_raw: List[Dict[str, Any]],
    *,
    sobject: str,
    gateway: SalesforceGateway,
    supported_fields: Optional[Set[str]],
    add_to_campaign: bool,
    default_campaign_status: Optional[str],
    campaign_validation_mode: str = CAMPAIGN_VALIDATION_FAIL_CLOSED,
    campaign_member_behavior: str = CAMPAIGN_MEMBER_BEHAVIOR_SKIP,
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]:
    """Process insert via REST Composite API (batches of 25)."""
    total_records = len(records_raw)
    results: List[Optional[Dict[str, Any]]] = [None] * total_records
    issues: List[Dict[str, Any]] = []
    success_count = 0
    campaign_assignments: List[Dict[str, Any]] = []
    valid_contexts: List[Dict[str, Any]] = []

    for idx, record in enumerate(records_raw):
        if not isinstance(record, dict):
            issues.append(
                {
                    "row": idx,
                    "code": "INVALID_RECORD",
                    "message": "Record must be an object.",
                }
            )
            results[idx] = {
                "success": False,
                "ok": False,
                "status": "invalid",
                "error": "Record must be an object.",
                "recordId": None,
                "id": None,
            }
            continue

        record_data = {k: v for k, v in record.items() if k != "attributes"}
        campaign_meta = _extract_campaign_metadata(record_data)
        normalized_data = _normalize_salesforce_data(
            sobject, record_data, supported_fields
        )
        unsupported_fields = _find_unsupported_fields(normalized_data, supported_fields)
        if unsupported_fields:
            _append_unsupported_field_issues(
                issues=issues,
                row=idx,
                sobject=sobject,
                fields=unsupported_fields,
                record_data=normalized_data,
            )
            results[idx] = {
                "success": False,
                "ok": False,
                "status": "unsupported_field",
                "error": (
                    f"Unsupported fields for {sobject}: "
                    + ", ".join(unsupported_fields)
                ),
                "errors": [
                    f"{field} is not available on {sobject} for this org."
                    for field in unsupported_fields
                ],
                "recordId": None,
                "id": None,
            }
            continue

        valid_contexts.append(
            {
                "index": idx,
                "normalized": normalized_data,
                "campaign_meta": campaign_meta,
            }
        )

    if not valid_contexts:
        final_results = [
            res
            if res is not None
            else {"success": False, "ok": False, "status": "skipped"}
            for res in results
        ]
        return final_results, issues, success_count

    batch_size = COMPOSITE_INSERT_BATCH_SIZE
    for start in range(0, len(valid_contexts), batch_size):
        batch = valid_contexts[start : start + batch_size]
        composite_request = []
        for ctx in batch:
            composite_request.append(
                {
                    "method": "POST",
                    "url": f"/services/data/{settings.SF_API_VERSION}/sobjects/{sobject}",
                    "referenceId": f"ref{ctx['index']}",
                    "body": ctx["normalized"],
                }
            )

        payload = {"allOrNone": False, "compositeRequest": composite_request}
        response = None
        for attempt in range(2):
            try:
                response = await gateway.composite(payload)
                break
            except SalesforceRateLimitError as exc:
                delay = exc.retry_after or COMPOSITE_RATE_LIMIT_BACKOFF_SECONDS
                log.warning(
                    "[SF IMPORT] Composite insert rate limited; retrying in %ss",
                    int(math.ceil(delay)),
                )
                await asyncio.sleep(delay)
                if attempt == 1:
                    raise
        composite_response = response.get("compositeResponse")

        if not isinstance(composite_response, list):
            raise BulkMappingError(
                "Composite response missing results for insert.",
                detail="Composite response did not include compositeResponse list.",
            )

        if len(composite_response) != len(batch):
            raise BulkMappingError(
                "Composite response count does not match request batch.",
                detail=(
                    f"Expected {len(batch)} results but received {len(composite_response)}."
                ),
            )

        rate_limited = False
        for offset, sub in enumerate(composite_response):
            ctx = batch[offset]
            idx = ctx["index"]
            campaign_meta = ctx["campaign_meta"]
            http_status = int(sub.get("httpStatusCode") or 0)
            body = sub.get("body")
            success = 200 <= http_status < 300

            if success:
                out_id = None
                if isinstance(body, dict):
                    out_id = body.get("id") or body.get("Id")
                if not out_id:
                    issues.append(
                        {
                            "row": idx,
                            "field": "Id",
                            "code": "MISSING_ID",
                            "message": "Composite insert response missing Id.",
                        }
                    )
                    result_entry = {
                        "success": False,
                        "ok": False,
                        "status": "failed",
                        "recordId": None,
                        "id": None,
                        "errors": [],
                        "error": "Composite insert response missing Id.",
                    }
                else:
                    success_count += 1
                    result_entry = {
                        "success": True,
                        "ok": True,
                        "status": "created",
                        "recordId": out_id,
                        "id": out_id,
                        "errors": [],
                        "error": None,
                    }
                    if add_to_campaign and campaign_meta["campaign_ids"]:
                        campaign_assignments.append(
                            {
                                "row": idx,
                                "record_id": out_id,
                                "campaign_ids": campaign_meta["campaign_ids"],
                                "status": campaign_meta["status"],
                            }
                        )
            else:
                errors = []
                if isinstance(body, list):
                    errors = body
                elif isinstance(body, dict):
                    errors = body.get("errors") or []
                message = "Salesforce error"
                code = "API_ERROR"
                field = None
                if errors:
                    err0 = errors[0] if isinstance(errors[0], dict) else {}
                    if isinstance(err0, dict):
                        message = err0.get("message") or message
                        code = err0.get("errorCode") or code
                        fields = err0.get("fields") or []
                        if fields:
                            field = fields[0]
                if (
                    code == "REQUEST_LIMIT_EXCEEDED"
                    or "REQUEST_LIMIT_EXCEEDED" in str(errors).upper()
                ):
                    rate_limited = True
                issues.append(
                    {
                        "row": idx,
                        "field": field,
                        "code": code,
                        "message": message,
                    }
                )
                result_entry = {
                    "success": False,
                    "ok": False,
                    "status": "failed",
                    "recordId": None,
                    "id": None,
                    "errors": errors,
                    "error": message,
                }

            if add_to_campaign and campaign_meta["campaign_ids"]:
                result_entry["campaignRequested"] = campaign_meta["campaign_ids"]

            results[idx] = result_entry

        if rate_limited:
            await asyncio.sleep(COMPOSITE_RATE_LIMIT_BACKOFF_SECONDS)

    final_results = [
        res
        if res is not None
        else {
            "success": False,
            "ok": False,
            "status": "skipped",
            "recordId": None,
            "id": None,
        }
        for res in results
    ]

    if add_to_campaign and campaign_assignments:
        await _process_campaign_memberships(
            gateway,
            sobject=sobject,
            assignments=campaign_assignments,
            default_status=default_campaign_status,
            results=final_results,
            issues=issues,
            validation_mode=campaign_validation_mode,
            duplicate_behavior=campaign_member_behavior,
        )

    return final_results, issues, success_count


async def _process_bulk_import(
    records_raw: List[Dict[str, Any]],
    *,
    sobject: str,
    operation: str,
    external_id_field: Optional[str],
    gateway: SalesforceGateway,
    supported_fields: Optional[Set[str]],
    add_to_campaign: bool,
    default_campaign_status: Optional[str],
    campaign_validation_mode: str = CAMPAIGN_VALIDATION_FAIL_CLOSED,
    campaign_member_behavior: str = CAMPAIGN_MEMBER_BEHAVIOR_SKIP,
    mapping_key: Optional[str],
    existing_job_id: Optional[str] = None,
    job_callback=None,
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]:
    """Process import using Salesforce Bulk API 2.0."""
    if not mapping_key:
        raise BulkMappingError(
            "Bulk results cannot be mapped deterministically without a key field.",
            detail="Use upsert with an external ID field or update with Id.",
        )
    mapping_key = mapping_key.strip()
    if not _FIELD_NAME_RE.match(mapping_key):
        raise BulkMappingError(
            "Bulk mapping key is invalid.",
            detail=f"Mapping key '{mapping_key}' is not a valid Salesforce field name.",
        )

    total_records = len(records_raw)
    results: List[Optional[Dict[str, Any]]] = [None] * total_records
    issues: List[Dict[str, Any]] = []
    success_count = 0

    valid_contexts: List[Dict[str, Any]] = []
    mapping_index: Dict[str, int] = {}

    for idx, record in enumerate(records_raw):
        if not isinstance(record, dict):
            issues.append(
                {
                    "row": idx,
                    "code": "INVALID_RECORD",
                    "message": "Record must be an object.",
                }
            )
            results[idx] = {
                "success": False,
                "ok": False,
                "status": "invalid",
                "error": "Record must be an object.",
                "recordId": None,
                "id": None,
            }
            continue

        record_data = {k: v for k, v in record.items() if k != "attributes"}
        record_id = record_data.pop("Id", None) or record_data.pop("id", None)
        campaign_meta = _extract_campaign_metadata(record_data)

        normalized_data = _normalize_salesforce_data(
            sobject, record_data, supported_fields
        )
        mapping_value: Optional[str] = None

        if operation == "update":
            if not record_id:
                issues.append(
                    {
                        "row": idx,
                        "code": "MISSING_ID",
                        "message": "Record is missing Id for update operation.",
                    }
                )
                results[idx] = {
                    "success": False,
                    "ok": False,
                    "status": "missing_id",
                    "recordId": None,
                    "id": None,
                    "error": "Missing Id for update",
                }
                continue
            normalized_data["Id"] = record_id
            mapping_value = _normalize_mapping_value(normalized_data.get("Id"))
        elif operation == "upsert":
            if not external_id_field:
                raise BulkMappingError(
                    "Bulk upsert requires an external ID field.",
                    detail="Provide external_id_field to use bulk upsert.",
                )
            if external_id_field not in normalized_data or not _has_value(
                normalized_data.get(external_id_field)
            ):
                issues.append(
                    {
                        "row": idx,
                        "field": external_id_field,
                        "code": "MISSING_EXTERNAL_ID",
                        "message": f"Upsert operation requires {external_id_field} field but value is missing.",
                    }
                )
                results[idx] = {
                    "success": False,
                    "ok": False,
                    "status": "skipped",
                    "recordId": None,
                    "id": None,
                    "error": f"Missing required field: {external_id_field}",
                }
                continue
            mapping_value = _normalize_mapping_value(
                normalized_data.get(external_id_field)
            )
        else:
            raise BulkMappingError(
                "Bulk operation is not supported for this request.",
                detail=f"Unsupported operation '{operation}' for bulk import.",
            )

        unsupported_fields = _find_unsupported_fields(normalized_data, supported_fields)
        if unsupported_fields:
            _append_unsupported_field_issues(
                issues=issues,
                row=idx,
                sobject=sobject,
                fields=unsupported_fields,
                record_data=normalized_data,
            )
            results[idx] = {
                "success": False,
                "ok": False,
                "status": "unsupported_field",
                "recordId": record_id,
                "id": record_id,
                "error": (
                    f"Unsupported fields for {sobject}: "
                    + ", ".join(unsupported_fields)
                ),
                "errors": [
                    f"{field} is not available on {sobject} for this org."
                    for field in unsupported_fields
                ],
            }
            continue

        if not mapping_value:
            issues.append(
                {
                    "row": idx,
                    "field": mapping_key,
                    "code": "MISSING_MAPPING_KEY",
                    "message": f"{mapping_key} is required for deterministic bulk mapping.",
                }
            )
            results[idx] = {
                "success": False,
                "ok": False,
                "status": "skipped",
                "recordId": record_id,
                "id": record_id,
                "error": f"Missing required field: {mapping_key}",
            }
            continue

        if mapping_value in mapping_index:
            duplicate_index = mapping_index[mapping_value]
            raise BulkMappingError(
                "Duplicate mapping key value in input rows.",
                detail=(
                    f"Duplicate {mapping_key} value '{mapping_value}' in rows "
                    f"{duplicate_index} and {idx}."
                ),
                issues=[
                    {
                        "row": idx,
                        "field": mapping_key,
                        "code": "DUPLICATE_MAPPING_KEY",
                        "message": (
                            f"Duplicate {mapping_key} value '{mapping_value}' in input."
                        ),
                    }
                ],
            )
        mapping_index[mapping_value] = idx

        valid_contexts.append(
            {
                "index": idx,
                "normalized": normalized_data,
                "record_id": record_id,
                "campaign_meta": campaign_meta,
                "mapping_value": mapping_value,
            }
        )

    if not valid_contexts:
        final_results = [
            res
            if res is not None
            else {"success": False, "ok": False, "status": "skipped"}
            for res in results
        ]
        return final_results, issues, success_count

    payload_records = [ctx["normalized"] for ctx in valid_contexts]

    try:
        if existing_job_id:
            bulk_result = await gateway.bulk_job_results(existing_job_id)
        elif operation == "insert":
            bulk_result = await gateway.bulk_create(
                sobject, payload_records, job_callback=job_callback
            )
        elif operation == "update":
            bulk_result = await gateway.bulk_update(
                sobject, payload_records, job_callback=job_callback
            )
        elif operation == "upsert":
            bulk_result = await gateway.bulk_upsert(
                sobject,
                payload_records,
                external_id_field,
                job_callback=job_callback,
            )
        else:
            raise BulkMappingError(
                "Bulk operation is not supported for this request.",
                detail=f"Unsupported operation '{operation}' for bulk import.",
            )
    except TimeoutError:
        raise
    except (BulkResultFetchError, BulkJobFailedError, BulkJobAbortedError):
        raise
    except Exception as exc:  # noqa: BLE001
        message, code, _ = _parse_salesforce_error(exc)
        raise HTTPException(
            status_code=500,
            detail={"error": code, "message": message},
        ) from exc

    success_results = bulk_result.get("successfulResults", [])
    failed_results = bulk_result.get("failedResults", [])
    unprocessed_results = bulk_result.get("unprocessedRecords", [])

    log.info(
        "[SF IMPORT] Bulk results: input=%s success=%s failed=%s unprocessed=%s",
        len(valid_contexts),
        len(success_results),
        len(failed_results),
        len(unprocessed_results),
    )

    def _extract_result_mapping_value(row: Dict[str, Any]) -> Optional[str]:
        raw = _get_row_value_case_insensitive(row, mapping_key)
        if (raw in (None, "") or not _has_value(raw)) and mapping_key.lower() == "id":
            raw = row.get("sf__Id")
        return _normalize_mapping_value(raw)

    result_by_key: Dict[str, tuple[str, Dict[str, Any]]] = {}
    mapping_failures: List[Dict[str, Any]] = []

    def _register_result(status: str, row: Dict[str, Any]) -> None:
        key_value = _extract_result_mapping_value(row)
        if not key_value:
            mapping_failures.append(
                {
                    "row": -1,
                    "field": mapping_key,
                    "code": "MISSING_RESULT_KEY",
                    "message": (
                        f"Bulk result missing {mapping_key} value; cannot map result."
                    ),
                }
            )
            return
        if key_value not in mapping_index:
            mapping_failures.append(
                {
                    "row": -1,
                    "field": mapping_key,
                    "code": "UNMAPPED_RESULT",
                    "message": (
                        f"Bulk result with {mapping_key}={key_value} does not match any input row."
                    ),
                }
            )
            return
        if key_value in result_by_key:
            mapping_failures.append(
                {
                    "row": mapping_index.get(key_value, -1),
                    "field": mapping_key,
                    "code": "DUPLICATE_RESULT_KEY",
                    "message": (
                        f"Bulk results contain duplicate {mapping_key} value '{key_value}'."
                    ),
                }
            )
            return
        result_by_key[key_value] = (status, row)

    for row in success_results:
        _register_result("success", row)
    for row in failed_results:
        _register_result("failed", row)
    for row in unprocessed_results:
        _register_result("unprocessed", row)

    if mapping_failures:
        raise BulkMappingError(
            "Cannot map bulk results to input rows deterministically.",
            detail="Bulk results contained unmapped or duplicate keys.",
            issues=mapping_failures[:10],
        )

    if len(result_by_key) != len(valid_contexts):
        raise BulkMappingError(
            "Bulk results count does not match input rows.",
            detail=(
                f"Expected {len(valid_contexts)} results but received {len(result_by_key)}."
            ),
            issues=[
                {
                    "row": -1,
                    "field": mapping_key,
                    "code": "RESULT_COUNT_MISMATCH",
                    "message": "Bulk results count does not match input rows.",
                }
            ],
        )

    campaign_assignments: List[Dict[str, Any]] = []

    for ctx in valid_contexts:
        idx = ctx["index"]
        record_id = ctx["record_id"]
        campaign_meta = ctx["campaign_meta"]
        mapping_value = ctx["mapping_value"]
        result_entry: Dict[str, Any]

        status, row = result_by_key[mapping_value]

        if status == "success":
            sf_id = row.get("sf__Id") or row.get("Id") or record_id
            created_flag = str(row.get("sf__Created")).lower()
            status_label = (
                "created"
                if operation == "insert" or created_flag == "true"
                else "updated"
            )
            result_entry = {
                "success": True,
                "ok": True,
                "status": status_label,
                "recordId": sf_id,
                "id": sf_id,
                "errors": [],
                "error": None,
            }
            success_count += 1
            if add_to_campaign and campaign_meta["campaign_ids"] and sf_id:
                campaign_assignments.append(
                    {
                        "row": idx,
                        "record_id": sf_id,
                        "campaign_ids": campaign_meta["campaign_ids"],
                        "status": campaign_meta["status"],
                    }
                )
        elif status == "failed":
            message = (
                row.get("sf__Error")
                or row.get("sf__ErrorMessage")
                or "Salesforce error"
            )
            code = row.get("sf__ErrorCode") or "API_ERROR"
            issues.append(
                {
                    "row": idx,
                    "field": row.get("sf__Field"),
                    "code": code,
                    "message": message,
                }
            )
            result_entry = {
                "success": False,
                "ok": False,
                "status": "failed",
                "recordId": record_id,
                "id": record_id,
                "errors": [{"statusCode": code, "message": message}],
                "error": message,
            }
        elif status == "unprocessed":
            message = row.get("sf__Error") or "Record unprocessed"
            code = row.get("sf__ErrorCode") or "UNPROCESSED"
            issues.append(
                {
                    "row": idx,
                    "field": row.get("sf__Field"),
                    "code": code,
                    "message": message,
                }
            )
            result_entry = {
                "success": False,
                "ok": False,
                "status": "unprocessed",
                "recordId": record_id,
                "id": record_id,
                "errors": [{"statusCode": code, "message": message}],
                "error": message,
            }
        else:
            raise BulkMappingError(
                "Bulk result status could not be mapped deterministically.",
                detail=f"Unknown result status '{status}'.",
            )

        if add_to_campaign and campaign_meta["campaign_ids"]:
            result_entry["campaignRequested"] = campaign_meta["campaign_ids"]

        results[idx] = result_entry

    final_results = [
        res
        if res is not None
        else {
            "success": False,
            "ok": False,
            "status": "skipped",
            "recordId": None,
            "id": None,
        }
        for res in results
    ]

    if add_to_campaign and campaign_assignments:
        await _process_campaign_memberships(
            gateway,
            sobject=sobject,
            assignments=campaign_assignments,
            default_status=default_campaign_status,
            results=final_results,
            issues=issues,
            validation_mode=campaign_validation_mode,
            duplicate_behavior=campaign_member_behavior,
        )

    return final_results, issues, success_count


@router.post("/bulk/delete")
async def bulk_salesforce_delete(
    payload: Dict[str, Any],
    api_ctx=Depends(require_api_key),
    db: AsyncSession = Depends(get_session),
    idem_key: Optional[str] = Header(default=None, alias="Idempotency-Key"),
):
    """Delete a batch of Salesforce records (used by rollback flow)."""
    ids = payload.get("ids")
    if not isinstance(ids, list) or not ids:
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_PAYLOAD", "message": "ids must be a non-empty list"},
        )

    object_type = str(payload.get("object") or payload.get("objectType") or "").strip()
    if not object_type or not _SOBJECT_NAME_RE.match(object_type):
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_OBJECT", "message": "Unsupported Salesforce object"},
        )

    gateway = await SalesforceGateway.for_tenant(api_ctx.account_id, db)

    results: List[Dict[str, Any]] = []
    issues: List[Dict[str, Any]] = []
    success_count = 0

    for idx, record_id in enumerate(ids):
        record_id = str(record_id or "").strip()
        if not record_id:
            issues.append(
                {
                    "row": idx,
                    "field": "Id",
                    "code": "MISSING_ID",
                    "message": "Record id is required for delete.",
                }
            )
            results.append(
                {
                    "success": False,
                    "ok": False,
                    "status": "missing_id",
                    "recordId": None,
                    "id": None,
                    "error": "Missing Id for delete",
                }
            )
            continue

        try:
            resp = await gateway.delete(object_type, record_id)
            success = bool(resp.get("success", False))
            if success:
                success_count += 1
            else:
                message = resp.get("message") or "Salesforce returned an error."
                issues.append(
                    {
                        "row": idx,
                        "field": "Id",
                        "code": resp.get("statusCode") or "API_ERROR",
                        "message": message,
                        "value": record_id,
                    }
                )
            results.append(
                {
                    "success": success,
                    "ok": success,
                    "status": "deleted" if success else "failed",
                    "recordId": record_id,
                    "id": record_id,
                    "errors": resp.get("errors") or [],
                    "error": None if success else resp.get("message"),
                }
            )
        except SalesforceRateLimitError as exc:
            headers: Dict[str, str] = {}
            if exc.retry_after:
                headers["Retry-After"] = str(int(math.ceil(exc.retry_after)))
            raise HTTPException(
                status_code=429,
                detail={"error": "RATE_LIMIT", "message": str(exc)},
                headers=headers or None,
            )
        except SalesforceAuthError as exc:
            raise HTTPException(
                status_code=401,
                detail={"error": "AUTH_ERROR", "message": str(exc)},
            )
        except SalesforceConnectionError as exc:
            raise HTTPException(
                status_code=503,
                detail={"error": "SFDC_UNAVAILABLE", "message": str(exc)},
            )
        except Exception as exc:  # noqa: BLE001
            message, code, field = _parse_salesforce_error(exc)
            issues.append(
                {
                    "row": idx,
                    "field": field or "Id",
                    "code": code,
                    "message": message,
                    "value": record_id,
                }
            )
            results.append(
                {
                    "success": False,
                    "ok": False,
                    "status": "failed",
                    "recordId": record_id,
                    "id": record_id,
                    "errors": [],
                    "error": message,
                }
            )

    response_body = {
        "ok": len(issues) == 0,
        "status": "completed" if len(issues) == 0 else "partial_success",
        "results": results,
        "issues": issues,
        "deleted": success_count,
        "errors": len(issues),
        "total": len(ids),
    }

    if idem_key:
        try:
            await idem_store.set(api_ctx.api_key_prefix, idem_key, response_body)
        except Exception:  # pragma: no cover - defensive
            log.warning("Failed to persist idempotency payload for key %s", idem_key)

    response = JSONResponse(content=response_body)
    response.headers["Cache-Control"] = "no-store"
    return response


@router.post("/bulk/{object_type}")
async def bulk_salesforce_import(
    object_type: str,
    payload: Dict[str, Any],
    api_ctx=Depends(require_api_key),
    db: AsyncSession = Depends(get_session),
    idem_key: Optional[str] = Header(default=None, alias="Idempotency-Key"),
):
    """
    Accept Salesforce import chunks from the Sheets add-on and perform insert/update/upsert operations.
    """
    records_raw = payload.get("records")
    if not isinstance(records_raw, list):
        raise HTTPException(
            status_code=400,
            detail={"error": "INVALID_PAYLOAD", "message": "records must be a list"},
        )

    options = payload.get("options") or {}
    operation = str(payload.get("operation") or "insert").lower()
    if operation not in {"insert", "update", "upsert"}:
        operation = "insert"

    dry_run = bool(options.get("dry_run"))
    external_id_field = (
        payload.get("external_id_field")
        or payload.get("externalIdField")
        or options.get("external_id_field")
        or options.get("externalIdField")
    )
    chunk_index_raw = (
        payload.get("chunk_index")
        or payload.get("chunkIndex")
        or options.get("chunk_index")
        or options.get("chunkIndex")
    )
    try:
        chunk_index = int(chunk_index_raw)
    except (TypeError, ValueError):
        chunk_index = 0
    job_id = str(
        payload.get("job_id")
        or payload.get("jobId")
        or idem_key
        or ("sf-import:" + uuid.uuid4().hex)
    )

    chunk_record_count = len(records_raw)
    total_records_hint = (
        payload.get("total_records")
        or payload.get("totalRecords")
        or options.get("total_records")
        or options.get("totalRecords")
        or options.get("total_rows")
        or options.get("totalRows")
    )
    try:
        total_records_candidate = int(total_records_hint)
    except (TypeError, ValueError):
        total_records_candidate = 0
    if total_records_candidate and total_records_candidate > 0:
        total_records = max(chunk_record_count, total_records_candidate)
    else:
        total_records = chunk_record_count
    chunk_size_hint = (
        payload.get("chunk_size")
        or payload.get("chunkSize")
        or options.get("chunk_size")
        or options.get("chunkSize")
    )
    try:
        configured_chunk_size = int(chunk_size_hint)
    except (TypeError, ValueError):
        configured_chunk_size = None
    chunk_size_effective = (
        configured_chunk_size
        if configured_chunk_size and configured_chunk_size > 0
        else chunk_record_count
    )
    chunk_count_estimate = max(
        1, math.ceil(total_records / max(1, chunk_size_effective))
    )
    log.info(
        "[SF IMPORT] job_id=%s, chunk_index=%s/%s, sobject=%s, operation=%s, dry_run=%s, chunk=%s, total=%s, external_id_field=%s",
        job_id,
        chunk_index if chunk_index > 0 else "-",
        chunk_count_estimate,
        object_type,
        operation,
        dry_run,
        chunk_record_count,
        total_records,
        external_id_field,
    )

    sobject_candidate: Optional[str] = None
    if records_raw:
        first = records_raw[0]
        if isinstance(first, dict):
            attrs = first.get("attributes")
            if isinstance(attrs, dict):
                sobject_candidate = attrs.get("type")

    sobject = (sobject_candidate or object_type or "").strip()
    if not sobject or not _SOBJECT_NAME_RE.match(sobject):
        raise HTTPException(
            status_code=400,
            detail={
                "error": "INVALID_OBJECT",
                "message": "Unsupported Salesforce object",
            },
        )

    if external_id_field:
        external_id_field = external_id_field.strip()
        if not _FIELD_NAME_RE.match(external_id_field):
            raise HTTPException(
                status_code=400,
                detail={
                    "error": "INVALID_EXTERNAL_ID_FIELD",
                    "message": f"External Id field '{external_id_field}' is not valid.",
                },
            )

    if operation == "upsert" and not external_id_field:
        raise BulkMappingError(
            "Upsert requires an external ID field.",
            detail="Provide external_id_field to perform upsert.",
        )

    gateway = await SalesforceGateway.for_tenant(api_ctx.account_id, db)

    use_bulk_api = bool(
        options.get("use_bulk_api")
        or options.get("useBulkApi")
        or options.get("bulk_api")
        or options.get("bulkApi")
        or payload.get("use_bulk_api")
        or payload.get("useBulkApi")
    )
    resolved_mode_hint = (
        options.get("resolved_mode")
        or options.get("resolvedMode")
        or payload.get("resolved_mode")
        or payload.get("resolvedMode")
    )
    force_rest_api = bool(
        options.get("force_rest_api")
        or options.get("forceRestApi")
        or options.get("force_rest")
        or options.get("forceRest")
    )
    requested_bulk_flag = use_bulk_api
    add_to_campaign = bool(
        options.get("add_to_campaign")
        or options.get("addToCampaign")
        or payload.get("add_to_campaign")
        or payload.get("addToCampaign")
    )
    campaign_status_raw = (
        options.get("campaign_status")
        or options.get("campaignStatus")
        or payload.get("campaign_status")
        or payload.get("campaignStatus")
    )
    campaign_status = (campaign_status_raw or "Responded").strip() or "Responded"
    campaign_validation_mode_raw = (
        options.get("campaign_validation_mode")
        or options.get("campaignValidationMode")
        or options.get("validation_mode")
        or options.get("validationMode")
        or payload.get("campaign_validation_mode")
        or payload.get("campaignValidationMode")
        or payload.get("validation_mode")
        or payload.get("validationMode")
    )
    campaign_validation_mode = _normalize_campaign_validation_mode(
        campaign_validation_mode_raw
    )
    campaign_member_behavior_raw = (
        options.get("campaign_member_behavior")
        or options.get("campaignMemberBehavior")
        or options.get("campaign_member_mode")
        or options.get("campaignMemberMode")
        or payload.get("campaign_member_behavior")
        or payload.get("campaignMemberBehavior")
        or payload.get("campaign_member_mode")
        or payload.get("campaignMemberMode")
    )
    campaign_member_behavior = _normalize_campaign_member_behavior(
        campaign_member_behavior_raw
    )
    campaign_source_raw = (
        options.get("campaign_source")
        or options.get("campaignSource")
        or options.get("campaign_mode")
        or options.get("campaignMode")
        or payload.get("campaign_source")
        or payload.get("campaignSource")
        or payload.get("campaign_mode")
        or payload.get("campaignMode")
    )
    campaign_source = None
    if campaign_source_raw:
        campaign_source = str(campaign_source_raw).strip().lower()
        if campaign_source not in {"ui", "columns"}:
            campaign_source = None
    campaign_id = (
        options.get("campaign_id")
        or options.get("campaignId")
        or payload.get("campaign_id")
        or payload.get("campaignId")
    )
    if campaign_id is not None:
        campaign_id = str(campaign_id).strip() or None

    if add_to_campaign and not _record_supports_campaign(sobject):
        log.warning(
            "[SF IMPORT] campaign enrollment requested for unsupported object %s; ignoring flag",
            sobject,
        )
        add_to_campaign = False
    if not add_to_campaign:
        campaign_source = None
        campaign_id = None

    if force_rest_api:
        use_bulk_api = False

    org_id = gateway.org_id or f"acct:{api_ctx.account_id}"
    if not gateway.org_id:
        log.warning(
            "[SF IMPORT] Salesforce org id missing for account %s; using account id fallback",
            api_ctx.account_id,
        )

    hash_options = dict(options or {})
    hash_options.update(
        {
            "_operation": operation,
            "_object": sobject,
            "_external_id_field": external_id_field,
            "_dry_run": dry_run,
            "_add_to_campaign": add_to_campaign,
            "_campaign_status": campaign_status,
            "_campaign_validation_mode": campaign_validation_mode,
            "_campaign_member_behavior": campaign_member_behavior,
            "_campaign_source": campaign_source,
            "_use_bulk_api": use_bulk_api,
            "_force_rest_api": force_rest_api,
        }
    )
    payload_hash = sfdc_idem.compute_payload_hash(records_raw, hash_options)

    can_use_bulk, bulk_mapping_key = _can_map_bulk_results_deterministically(
        operation, records_raw, external_id_field
    )
    use_composite_insert = operation == "insert" and not can_use_bulk
    warnings = _compute_import_warnings(
        records_raw,
        use_composite_insert=use_composite_insert,
        add_to_campaign=add_to_campaign,
        tier=api_ctx.tier,
        dry_run=dry_run,
    )

    cached_response_body: Optional[Dict[str, Any]] = None
    if idem_key:
        cached_payload = await idem_store.get(api_ctx.api_key_prefix, idem_key)
        if isinstance(cached_payload, dict):
            meta = cached_payload.get("_idem_meta")
            if isinstance(meta, dict) and meta.get("payload_hash") == payload_hash:
                if (
                    meta.get("import_run_id") == job_id
                    and meta.get("chunk_index") == chunk_index
                    and meta.get("org_id") == org_id
                ):
                    cached_response_body = (
                        cached_payload.get("response") or cached_payload
                    )
    if cached_response_body:
        cached_body = dict(cached_response_body)
        if "_idem_meta" in cached_body:
            response_body = cached_body.get("response")
            if isinstance(response_body, dict):
                cached_body = dict(response_body)
            cached_body.pop("_idem_meta", None)
        cached_body["cached"] = True
        cached_body.setdefault("job_id", job_id)
        _attach_warnings(cached_body, warnings)
        cached_response = JSONResponse(content=cached_body)
        cached_response.headers["Cache-Control"] = "no-store"
        return cached_response

    chunk_from_idem = None
    if idem_key:
        chunk_from_idem = await sfdc_idem.get_chunk_by_idempotency_key(
            db, idem_key, org_id
        )
        if chunk_from_idem and chunk_from_idem.payload_hash != payload_hash:
            response_body = {
                "ok": False,
                "status": "payload_drift",
                "error": "Chunk payload changed since the original request.",
                "details": (
                    "Payload hash does not match the request for this idempotency key. "
                    "Start a new import run to continue."
                ),
                "processed": 0,
                "errors": 1,
                "total": total_records,
                "dry_run": dry_run,
                "mode": "rest",
                "chunk_size": chunk_size_effective,
                "chunk_rows": chunk_record_count,
                "chunk_count": chunk_count_estimate,
                "requested_bulk": requested_bulk_flag,
                "effective_bulk": use_bulk_api,
                "force_rest": force_rest_api,
                "mapping_method": chunk_from_idem.mapping_method,
                "mapping_key": None,
                "batch_size": chunk_size_effective,
                "cached": True,
            }
            _attach_warnings(response_body, warnings)
            response = JSONResponse(content=response_body, status_code=409)
            response.headers["Cache-Control"] = "no-store"
            return response
        if chunk_from_idem:
            job_id = chunk_from_idem.import_run_id

    if chunk_from_idem is None:
        existing_identity = await sfdc_idem.get_latest_chunk_for_index(
            db,
            org_id=org_id,
            import_run_id=job_id,
            chunk_index=chunk_index,
        )
        if existing_identity and existing_identity.payload_hash != payload_hash:
            response_body = {
                "ok": False,
                "status": "payload_drift",
                "error": "Chunk payload changed since the original request.",
                "details": (
                    "Payload hash does not match previously processed chunk. "
                    "Start a new import run to continue."
                ),
                "processed": 0,
                "errors": 1,
                "total": total_records,
                "dry_run": dry_run,
                "mode": "rest",
                "chunk_size": chunk_size_effective,
                "chunk_rows": chunk_record_count,
                "chunk_count": chunk_count_estimate,
                "requested_bulk": requested_bulk_flag,
                "effective_bulk": use_bulk_api,
                "force_rest": force_rest_api,
                "mapping_method": existing_identity.mapping_method,
                "mapping_key": None,
                "batch_size": chunk_size_effective,
                "cached": True,
            }
            _attach_warnings(response_body, warnings)
            response = JSONResponse(content=response_body, status_code=409)
            response.headers["Cache-Control"] = "no-store"
            return response

        chunk, _ = await sfdc_idem.get_or_create_chunk(
            db,
            org_id=org_id,
            import_run_id=job_id,
            chunk_index=chunk_index,
            payload_hash=payload_hash,
            idempotency_key=idem_key,
        )
        if idem_key and not chunk.idempotency_key:
            chunk.idempotency_key = idem_key
            await db.flush()
    else:
        chunk = chunk_from_idem

    log.debug(
        "[SF IMPORT] Mode resolution: requested_bulk=%s, effective_bulk=%s, force_rest=%s, resolved_hint=%s, chunk=%s, total=%s",
        requested_bulk_flag,
        use_bulk_api,
        force_rest_api,
        resolved_mode_hint or "-",
        chunk_record_count,
        total_records,
    )

    wants_bulk = (
        (not dry_run)
        and (not force_rest_api)
        and (use_bulk_api or total_records >= BULK_API_THRESHOLD)
    )

    if wants_bulk and not can_use_bulk:
        log.info(
            "[SF IMPORT] Bulk requested but operation '%s' cannot be mapped deterministically; "
            "routing to composite insert",
            operation,
        )

    resume_job_id: Optional[str] = None
    if chunk.status == "completed":
        cached_payload = sfdc_idem.decompress_results(chunk)
        results_cached = list(cached_payload.get("results") or [])
        issues_cached = list(cached_payload.get("issues") or [])
        summary = chunk.result_summary or {}
        success_cached = summary.get("success_count")
        if success_cached is None:
            success_cached = sum(
                1 for row in results_cached if row.get("success") is True
            )
        cached_mapping_method = chunk.mapping_method or "rest_by_index"
        cached_mapping_key = _mapping_key_from_method(
            cached_mapping_method, external_id_field
        )
        cached_batch_size = (
            COMPOSITE_INSERT_BATCH_SIZE
            if cached_mapping_method == "composite_by_index"
            else chunk_size_effective
        )
        cached_execution_mode = _execution_mode_from_mapping(
            cached_mapping_method, dry_run=dry_run
        )
        cached_status = _derive_import_status(issues_cached)
        response_body = {
            "ok": cached_status == "completed",
            "status": cached_status,
            "job_id": job_id,
            "results": results_cached,
            "issues": issues_cached,
            "processed": int(success_cached or 0),
            "errors": len(issues_cached),
            "total": total_records,
            "dry_run": dry_run,
            "mode": cached_execution_mode,
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": cached_mapping_method,
            "mapping_key": cached_mapping_key,
            "batch_size": cached_batch_size,
            "cached": True,
        }
        _attach_warnings(response_body, warnings)
        response = JSONResponse(content=response_body)
        response.headers["Cache-Control"] = "no-store"
        return response

    if chunk.status == "failed":
        cached_payload = sfdc_idem.decompress_results(chunk)
        results_cached = list(cached_payload.get("results") or [])
        issues_cached = list(cached_payload.get("issues") or [])
        summary = chunk.result_summary or {}
        cached_mapping_method = chunk.mapping_method or "rest_by_index"
        cached_mapping_key = _mapping_key_from_method(
            cached_mapping_method, external_id_field
        )
        cached_batch_size = (
            COMPOSITE_INSERT_BATCH_SIZE
            if cached_mapping_method == "composite_by_index"
            else chunk_size_effective
        )
        cached_execution_mode = _execution_mode_from_mapping(
            cached_mapping_method, dry_run=dry_run
        )
        status_label = summary.get("status") or "failed"
        error_message = summary.get("error") or "Import failed."
        response_body = {
            "ok": False,
            "status": status_label,
            "error": error_message,
            "job_id": job_id,
            "salesforce_job_id": chunk.salesforce_job_id,
            "can_resume": False,
            "results": results_cached,
            "issues": issues_cached,
            "processed": 0,
            "errors": len(issues_cached) or 1,
            "total": total_records,
            "dry_run": dry_run,
            "mode": cached_execution_mode,
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": cached_mapping_method,
            "mapping_key": cached_mapping_key,
            "batch_size": cached_batch_size,
            "cached": True,
        }
        _attach_warnings(response_body, warnings)
        status_code = summary.get("http_status")
        if not status_code:
            status_code = 409 if status_label == "mapping_failed" else 500
        response = JSONResponse(content=response_body, status_code=status_code)
        response.headers["Cache-Control"] = "no-store"
        return response

    if chunk.status == "processing":
        if chunk.salesforce_job_id:
            resume_job_id = chunk.salesforce_job_id
            log.info(
                "[SF IMPORT] Resuming poll for bulk job %s (chunk %s/%s)",
                resume_job_id,
                job_id,
                chunk_index,
            )
        else:
            updated_at = chunk.updated_at or chunk.created_at or datetime.now(
                timezone.utc
            )
            if updated_at.tzinfo is None:
                updated_at = updated_at.replace(tzinfo=timezone.utc)
            age = datetime.now(timezone.utc) - updated_at
            if age <= timedelta(minutes=5):
                response_body = {
                    "ok": True,
                    "status": "processing",
                    "message": "Import chunk is currently being processed",
                    "retry_after_seconds": 10,
                    "job_id": job_id,
                    "salesforce_job_id": chunk.salesforce_job_id,
                    "can_resume": bool(chunk.salesforce_job_id),
                    "total": total_records,
                    "dry_run": dry_run,
                    "chunk_size": chunk_size_effective,
                    "chunk_rows": chunk_record_count,
                    "chunk_count": chunk_count_estimate,
                    "requested_bulk": requested_bulk_flag,
                    "effective_bulk": use_bulk_api,
                    "force_rest": force_rest_api,
                    "mapping_method": chunk.mapping_method,
                    "mapping_key": _mapping_key_from_method(
                        chunk.mapping_method, external_id_field
                    ),
                    "batch_size": chunk_size_effective,
                    "cached": True,
                }
                _attach_warnings(response_body, warnings)
                response = JSONResponse(content=response_body)
                response.headers["Cache-Control"] = "no-store"
                return response

            log.warning(
                "[SF IMPORT] Stale processing chunk detected (age=%s); resetting to pending",
                age,
            )
            chunk.status = "pending"
            chunk.updated_at = datetime.now(timezone.utc)
            await db.commit()

    required_fields: List[str] = []
    supported_fields: Optional[Set[str]] = None
    try:
        describe = await gateway.describe(sobject)
        describe_fields = [
            field
            for field in (describe.get("fields", []) if isinstance(describe, dict) else [])
            if isinstance(field, dict)
        ]
        supported_fields = {
            str(field.get("name"))
            for field in describe_fields
            if field.get("name")
        }
        if dry_run:
            required_fields = [
                str(field["name"])
                for field in describe_fields
                if field.get("name")
                and not field.get("nillable", True)
                and not field.get("defaultedOnCreate", False)
                and not field.get("autoNumber", False)
            ]
    except Exception as exc:  # pragma: no cover - defensive
        log.warning("Failed to fetch describe metadata for %s: %s", sobject, exc)

    if (
        external_id_field
        and supported_fields is not None
        and external_id_field not in supported_fields
    ):
        raise HTTPException(
            status_code=400,
            detail={
                "error": "INVALID_EXTERNAL_ID_FIELD",
                "message": (
                    f"External Id field '{external_id_field}' is not available on "
                    f"{sobject} for this org."
                ),
            },
        )

    results: List[Dict[str, Any]]
    issues: List[Dict[str, Any]]
    success_count: int
    mapping_method = "rest_by_index"
    mapping_key_used: Optional[str] = None
    batch_size = chunk_size_effective

    try:
        if resume_job_id:
            mapping_method = chunk.mapping_method or (
                "bulk_by_id"
                if (bulk_mapping_key or "").lower() == "id"
                else "bulk_by_external_id"
            )
            mapping_key_used = _mapping_key_from_method(
                mapping_method, external_id_field
            )
            if not mapping_key_used:
                raise BulkMappingError(
                    "Cannot resume bulk job without a deterministic mapping key.",
                    detail="Provide Id or external ID field to resume bulk results.",
                )
            await sfdc_idem.mark_processing(
                db,
                chunk,
                salesforce_job_id=resume_job_id,
                mapping_method=mapping_method,
            )
            await db.commit()
            results, issues, success_count = await _process_bulk_import(
                records_raw,
                sobject=sobject,
                operation=operation,
                external_id_field=external_id_field,
                gateway=gateway,
                supported_fields=supported_fields,
                add_to_campaign=add_to_campaign,
                default_campaign_status=campaign_status,
                campaign_validation_mode=campaign_validation_mode,
                campaign_member_behavior=campaign_member_behavior,
                mapping_key=mapping_key_used,
                existing_job_id=resume_job_id,
            )
        elif dry_run:
            log.info("[SF IMPORT] Using REST API (dry run)")
            mapping_method = "rest_by_index"
            await sfdc_idem.mark_processing(
                db,
                chunk,
                mapping_method=mapping_method,
            )
            await db.commit()
            results, issues, success_count = await _process_rest_import(
                records_raw,
                sobject=sobject,
                operation=operation,
                external_id_field=external_id_field,
                gateway=gateway,
                dry_run=True,
                required_fields=required_fields,
                supported_fields=supported_fields,
                add_to_campaign=add_to_campaign,
                default_campaign_status=campaign_status,
                campaign_validation_mode=campaign_validation_mode,
                campaign_member_behavior=campaign_member_behavior,
            )
        elif operation == "insert" and not can_use_bulk:
            log.info(
                "[SF IMPORT] Using Composite API path for keyless insert (chunk=%s, total=%s)",
                chunk_record_count,
                total_records,
            )
            mapping_method = "composite_by_index"
            batch_size = COMPOSITE_INSERT_BATCH_SIZE
            await sfdc_idem.mark_processing(
                db,
                chunk,
                mapping_method=mapping_method,
            )
            await db.commit()
            results, issues, success_count = await _process_composite_insert(
                records_raw,
                sobject=sobject,
                gateway=gateway,
                supported_fields=supported_fields,
                add_to_campaign=add_to_campaign,
                default_campaign_status=campaign_status,
                campaign_validation_mode=campaign_validation_mode,
                campaign_member_behavior=campaign_member_behavior,
            )
        elif wants_bulk and can_use_bulk:
            log.info(
                "[SF IMPORT] Using Bulk API path (chunk=%s, configured_chunk=%s, total=%s, threshold=%s, forced=%s, resolved_hint=%s)",
                chunk_record_count,
                chunk_size_effective,
                total_records,
                BULK_API_THRESHOLD,
                use_bulk_api,
                resolved_mode_hint or "-",
            )
            mapping_key_used = bulk_mapping_key
            mapping_method = (
                "bulk_by_id"
                if (bulk_mapping_key or "").lower() == "id"
                else "bulk_by_external_id"
            )
            await sfdc_idem.mark_processing(
                db,
                chunk,
                mapping_method=mapping_method,
            )
            await db.commit()

            async def _record_bulk_job(new_job_id: str) -> None:
                await sfdc_idem.mark_processing(
                    db,
                    chunk,
                    salesforce_job_id=new_job_id,
                    mapping_method=mapping_method,
                )
                await db.commit()

            results, issues, success_count = await _process_bulk_import(
                records_raw,
                sobject=sobject,
                operation=operation,
                external_id_field=external_id_field,
                gateway=gateway,
                supported_fields=supported_fields,
                add_to_campaign=add_to_campaign,
                default_campaign_status=campaign_status,
                campaign_validation_mode=campaign_validation_mode,
                campaign_member_behavior=campaign_member_behavior,
                mapping_key=bulk_mapping_key,
                job_callback=_record_bulk_job,
            )
        else:
            log.info(
                "[SF IMPORT] Using REST API path (chunk=%s, configured_chunk=%s, total=%s, force_rest=%s)",
                chunk_record_count,
                chunk_size_effective,
                total_records,
                force_rest_api,
            )
            mapping_method = "rest_by_index"
            await sfdc_idem.mark_processing(
                db,
                chunk,
                mapping_method=mapping_method,
            )
            await db.commit()
            results, issues, success_count = await _process_rest_import(
                records_raw,
                sobject=sobject,
                operation=operation,
                external_id_field=external_id_field,
                gateway=gateway,
                dry_run=False,
                required_fields=required_fields,
                supported_fields=supported_fields,
                add_to_campaign=add_to_campaign,
                default_campaign_status=campaign_status,
                campaign_validation_mode=campaign_validation_mode,
                campaign_member_behavior=campaign_member_behavior,
            )

        await sfdc_idem.mark_completed(
            db,
            chunk,
            results=results,
            issues=issues,
            success_count=success_count,
            mapping_method=mapping_method,
        )
        await db.commit()
    except TimeoutError:
        processing_job_id = chunk.salesforce_job_id or resume_job_id
        await sfdc_idem.mark_processing(
            db,
            chunk,
            salesforce_job_id=processing_job_id,
            mapping_method=mapping_method,
        )
        await db.commit()
        response_body = {
            "ok": True,
            "status": "processing",
            "message": "Import chunk is currently being processed",
            "retry_after_seconds": 10,
            "job_id": job_id,
            "salesforce_job_id": processing_job_id,
            "can_resume": True,
            "total": total_records,
            "dry_run": dry_run,
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": mapping_method,
            "mapping_key": _mapping_key_from_method(
                mapping_method, external_id_field
            ),
            "batch_size": batch_size,
            "cached": True,
        }
        _attach_warnings(response_body, warnings)
        response = JSONResponse(content=response_body)
        response.headers["Cache-Control"] = "no-store"
        return response
    except BulkResultFetchError as exc:
        fetch_job_id = exc.job_id or chunk.salesforce_job_id or resume_job_id
        await sfdc_idem.mark_processing(
            db,
            chunk,
            salesforce_job_id=fetch_job_id,
            mapping_method=mapping_method,
        )
        await db.commit()
        response_body = {
            "ok": False,
            "status": "result_fetch_failed",
            "error": str(exc),
            "job_id": job_id,
            "salesforce_job_id": fetch_job_id,
            "can_resume": True,
            "guidance": (
                "Bulk job completed but result retrieval failed. Retry to fetch results."
            ),
            "processed": 0,
            "errors": 1,
            "total": total_records,
            "dry_run": dry_run,
            "mode": "rest",
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": mapping_method,
            "mapping_key": _mapping_key_from_method(
                mapping_method, external_id_field
            ),
            "batch_size": batch_size,
        }
        _attach_warnings(response_body, warnings)
        response = JSONResponse(content=response_body, status_code=502)
        response.headers["Cache-Control"] = "no-store"
        return response
    except BulkJobAbortedError as exc:
        aborted_job_id = exc.job_id or chunk.salesforce_job_id or resume_job_id
        error_msg = str(exc)
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=error_msg,
            mapping_method=mapping_method,
            status="aborted",
            http_status=409,
        )
        await db.commit()
        response_body = {
            "ok": False,
            "status": "aborted",
            "error": error_msg,
            "job_id": job_id,
            "salesforce_job_id": aborted_job_id,
            "can_resume": False,
            "guidance": (
                "Job was aborted by Salesforce. Check data size limits or org restrictions."
            ),
            "processed": 0,
            "errors": 1,
            "total": total_records,
            "dry_run": dry_run,
            "mode": "rest",
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": mapping_method,
            "mapping_key": _mapping_key_from_method(
                mapping_method, external_id_field
            ),
            "batch_size": batch_size,
        }
        _attach_warnings(response_body, warnings)
        response = JSONResponse(content=response_body, status_code=409)
        response.headers["Cache-Control"] = "no-store"
        return response
    except BulkJobFailedError as exc:
        failed_job_id = exc.job_id or chunk.salesforce_job_id or resume_job_id
        error_msg = str(exc)
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=error_msg,
            mapping_method=mapping_method,
            status="failed",
            http_status=500,
        )
        await db.commit()
        response_body = {
            "ok": False,
            "status": "failed",
            "error": error_msg,
            "job_id": job_id,
            "salesforce_job_id": failed_job_id,
            "can_resume": False,
            "processed": 0,
            "errors": 1,
            "total": total_records,
            "dry_run": dry_run,
            "mode": "rest",
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": mapping_method,
            "mapping_key": _mapping_key_from_method(
                mapping_method, external_id_field
            ),
            "batch_size": batch_size,
        }
        _attach_warnings(response_body, warnings)
        response = JSONResponse(content=response_body, status_code=500)
        response.headers["Cache-Control"] = "no-store"
        return response
    except SalesforceRateLimitError as exc:
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=str(exc),
            mapping_method=mapping_method,
            status="rate_limited",
            http_status=429,
        )
        await db.commit()
        headers: Dict[str, str] = {}
        if exc.retry_after:
            headers["Retry-After"] = str(int(math.ceil(exc.retry_after)))
        raise HTTPException(
            status_code=429,
            detail={"error": "RATE_LIMIT", "message": str(exc)},
            headers=headers or None,
        )
    except SalesforceAuthError as exc:
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=str(exc),
            mapping_method=mapping_method,
            status="auth_error",
            http_status=401,
        )
        await db.commit()
        raise HTTPException(
            status_code=401,
            detail={"error": "AUTH_ERROR", "message": str(exc)},
        )
    except SalesforceConnectionError as exc:
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=str(exc),
            mapping_method=mapping_method,
            status="sfdc_unavailable",
            http_status=503,
        )
        await db.commit()
        raise HTTPException(
            status_code=503,
            detail={"error": "SFDC_UNAVAILABLE", "message": str(exc)},
        )
    except BulkMappingError as exc:
        issues_out = list(exc.issues or [])
        if not issues_out:
            issues_out.append(
                {
                    "row": -1,
                    "field": mapping_key_used,
                    "code": "MAPPING_FAILED",
                    "message": exc.message,
                }
            )
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=exc.message,
            issues=issues_out,
            mapping_method=mapping_method,
            status="mapping_failed",
            http_status=409,
        )
        await db.commit()
        response_body = {
            "ok": False,
            "status": "mapping_failed",
            "error": exc.message,
            "details": exc.detail,
            "issues": issues_out,
            "processed": 0,
            "errors": len(issues_out),
            "total": total_records,
            "dry_run": dry_run,
            "mode": "rest",
            "chunk_size": chunk_size_effective,
            "chunk_rows": chunk_record_count,
            "chunk_count": chunk_count_estimate,
            "requested_bulk": requested_bulk_flag,
            "effective_bulk": use_bulk_api,
            "force_rest": force_rest_api,
            "mapping_method": mapping_method,
            "mapping_key": mapping_key_used,
            "batch_size": batch_size,
            "salesforce_job_id": chunk.salesforce_job_id,
            "can_resume": False,
        }
        _attach_warnings(response_body, warnings)
        response = JSONResponse(content=response_body, status_code=409)
        response.headers["Cache-Control"] = "no-store"
        return response
    except HTTPException as exc:
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=str(exc.detail or exc),
            mapping_method=mapping_method,
            status="failed",
            http_status=exc.status_code,
        )
        await db.commit()
        raise
    except Exception as exc:  # noqa: BLE001
        await sfdc_idem.mark_failed(
            db,
            chunk,
            error=str(exc),
            mapping_method=mapping_method,
            status="failed",
            http_status=500,
        )
        await db.commit()
        raise

    execution_mode = "rest"
    if not dry_run:
        execution_mode = _execution_mode_from_mapping(
            mapping_method, dry_run=dry_run
        )

    response_status = _derive_import_status(issues)
    response_body = {
        "ok": response_status == "completed",
        "status": response_status,
        "job_id": job_id,
        "results": results,
        "issues": issues,
        "processed": success_count,
        "errors": len(issues),
        "total": total_records,
        "dry_run": dry_run,
        "mode": execution_mode,
        "chunk_size": chunk_size_effective,
        "chunk_rows": chunk_record_count,
        "chunk_count": chunk_count_estimate,
        "requested_bulk": requested_bulk_flag,
        "effective_bulk": use_bulk_api,
        "force_rest": force_rest_api,
        "mapping_method": mapping_method,
        "mapping_key": mapping_key_used,
        "batch_size": batch_size,
        "campaign_source": campaign_source,
        "campaign_id": campaign_id if campaign_source == "ui" else None,
        "campaign_member_behavior": campaign_member_behavior,
    }
    _attach_warnings(response_body, warnings)

    if idem_key:
        cache_payload = {
            "_idem_meta": {
                "payload_hash": payload_hash,
                "import_run_id": job_id,
                "chunk_index": chunk_index,
                "org_id": org_id,
            },
            "response": response_body,
        }
        try:
            await idem_store.set(api_ctx.api_key_prefix, idem_key, cache_payload)
        except Exception:  # pragma: no cover - defensive
            log.warning("Failed to persist idempotency payload for key %s", idem_key)

    response = JSONResponse(content=response_body)
    response.headers["Cache-Control"] = "no-store"
    return response


@router.get("/campaigns")
async def list_salesforce_campaigns(
    q: str = Query(default="", max_length=80),
    limit: int = Query(default=25, ge=1, le=200),
    api_ctx=Depends(require_api_key),
    db: AsyncSession = Depends(get_session),
):
    """List active Salesforce campaigns (optionally filtered by name)."""
    gateway = await SalesforceGateway.for_tenant(api_ctx.account_id, db)
    term = re.sub(r"[^A-Za-z0-9 _-]", " ", (q or "")).strip()
    term = re.sub(r"\s+", " ", term)
    where_clause = "WHERE IsActive = true"
    if term:
        safe = term.replace("'", "\\'")
        where_clause += f" AND Name LIKE '%{safe}%'"
    query = (
        "SELECT Id, Name, Status, IsActive, StartDate, EndDate "
        f"FROM Campaign {where_clause} "
        "ORDER BY LastModifiedDate DESC "
        f"LIMIT {limit}"
    )
    results = await gateway.soql(query)
    items = []
    for record in results.get("records") or []:
        items.append(
            {
                "id": record.get("Id"),
                "name": record.get("Name"),
                "status": record.get("Status"),
                "is_active": record.get("IsActive"),
                "start_date": record.get("StartDate"),
                "end_date": record.get("EndDate"),
            }
        )
    return {"items": items, "total": len(items)}


@router.get("/campaigns/{campaign_id}/statuses")
async def list_campaign_member_statuses(
    campaign_id: str,
    api_ctx=Depends(require_api_key),
    db: AsyncSession = Depends(get_session),
):
    """List CampaignMember status values for a specific campaign."""
    campaign_id = (campaign_id or "").strip()
    if not _SF_ID_RE.fullmatch(campaign_id):
        raise HTTPException(
            status_code=400,
            detail={
                "error": "INVALID_CAMPAIGN_ID",
                "message": "Campaign Id must be a 15 or 18 character Salesforce ID.",
            },
        )
    gateway = await SalesforceGateway.for_tenant(api_ctx.account_id, db)
    query = (
        "SELECT Label, IsDefault "
        "FROM CampaignMemberStatus "
        f"WHERE CampaignId = '{campaign_id}' "
        "ORDER BY IsDefault DESC, SortOrder ASC"
    )
    results = await gateway.soql(query)
    statuses = []
    default_status = None
    for record in results.get("records") or []:
        label = record.get("Label")
        is_default = bool(record.get("IsDefault"))
        if label is None:
            continue
        if is_default and default_status is None:
            default_status = label
        statuses.append(
            {
                "label": label,
                "is_default": is_default,
            }
        )
    return {
        "campaign_id": campaign_id,
        "statuses": statuses,
        "default_status": default_status,
    }


@router.post("/campaign_members/push")
def push_campaign_members(
    payload: Dict[str, Any],
    api_ctx=Depends(require_api_key),
    idem: Optional[str] = Header(default=None, alias="Idempotency-Key"),
):
    """
    Accept reviewed rows and build CampaignMember records.
    Body: { headers, rows, campaignId, status, dry_run }
    """
    headers = list(payload.get("headers") or [])
    rows = list(payload.get("rows") or [])
    campaign_id = str(payload.get("campaignId") or "")
    status = str(payload.get("status") or "Responded")
    dry_run = bool(payload.get("dry_run", True))
    if not campaign_id:
        raise HTTPException(400, "campaignId required")

    H = {str(h): i for i, h in enumerate(headers)}
    need = ["FM_Status", "FM_Target_Type", "FM_Target_Id"]
    miss = [n for n in need if n not in H]
    if miss:
        raise HTTPException(400, "Missing required columns: " + ", ".join(miss))

    cms: List[Dict[str, str]] = []
    for r in rows:
        st = str(r[H["FM_Status"]] or "")
        if st != "Accepted":
            continue
        ttype = str(r[H["FM_Target_Type"]] or "")
        tid = str(r[H["FM_Target_Id"]] or "")
        if not tid:
            continue
        cm = {
            "CampaignId": campaign_id,
            "LeadId": "",
            "ContactId": "",
            "Status": status,
        }
        if ttype.lower() == "lead":
            cm["LeadId"] = tid
        elif ttype.lower() == "contact":
            cm["ContactId"] = tid
        else:
            continue
        cms.append(cm)

    if dry_run:
        res = {
            "ok": True,
            "dry_run": True,
            "counts": {"input_rows": len(rows), "cm_rows": len(cms)},
            "sample": cms[:5],
        }
        out = JSONResponse(content=res)
        out.headers["Cache-Control"] = "no-store"
        return out

    job_id = idem or ("cm-" + uuid.uuid4().hex)
    buf = io.StringIO()
    w = csv.DictWriter(buf, fieldnames=["CampaignId", "LeadId", "ContactId", "Status"])
    w.writeheader()
    for x in cms:
        w.writerow(x)
    _PUSH_JOBS[job_id] = {
        "status": "completed",
        "result_csv": buf.getvalue(),
        "created_at": time.time(),
        "account_id": str(api_ctx.account_id or ""),
    }
    res = JSONResponse(content={"ok": True, "dry_run": False, "job_id": job_id})
    res.headers["Cache-Control"] = "no-store"
    return res


@router.get("/campaign_members/push/{job_id}/results")
def push_results(job_id: str, api_ctx=Depends(require_api_key)):
    job = _PUSH_JOBS.get(job_id)
    if not job:
        raise HTTPException(404, "JOB_NOT_FOUND")
    account_id = str(getattr(api_ctx, "account_id", "") or "")
    if not account_id:
        raise HTTPException(401, "Missing account_id in API context")
    if str(job.get("account_id") or "") != account_id:
        raise HTTPException(404, "JOB_NOT_FOUND")
    res = JSONResponse(
        content={"ok": True, "status": job["status"], "csv": job.get("result_csv", "")}
    )
    res.headers["Cache-Control"] = "no-store"
    return res


@router.post("/reports")
async def list_salesforce_reports(
    payload: Dict[str, Any] = {},
    api_ctx=Depends(require_api_key),
):
    """
    List Salesforce reports with optional search/filtering.

    Body params:
    - q: Search query for report name
    - type: Filter by report type/folder (e.g., 'Accounts', 'Leads')
    - limit: Max results (default 25)
    - page_token: Pagination token
    """
    from ...db import get_session

    log.info(f"Reports API called with payload: {payload}")

    q = str(payload.get("q", ""))
    type_param = str(payload.get("type", ""))
    try:
        limit = int(payload.get("limit", 25))
    except (ValueError, TypeError):
        limit = 25
    page_token = str(payload.get("page_token", ""))

    # Get gateway for this tenant
    account_id = api_ctx.account_id
    if not account_id:
        raise HTTPException(401, "Missing account_id in API context")

    # Create DB session
    async for db in get_session():
        try:
            from ...services.salesforce_gateway import SalesforceGateway

            gateway = await SalesforceGateway.for_tenant(str(account_id), db)

            log.info(
                f"Listing reports for account {account_id}, query='{q}', type='{type_param}', limit={limit}"
            )

            # Call gateway method
            result = await gateway.list_reports(
                query=q, report_type=type_param, limit=limit, page_token=page_token
            )

            log.info(f"Reports API returned {len(result.get('items', []))} items")

            return JSONResponse(content={"ok": True, "data": result})
        except Exception as e:
            log.error(f"Failed to list reports: {e}", exc_info=True)
            raise HTTPException(500, f"Failed to list reports: {str(e)}")
        finally:
            await db.close()


@router.post("/report/export_batch")
async def export_salesforce_reports_batch(
    payload: Dict[str, Any],
    api_ctx=Depends(require_api_key),
):
    """Export multiple Salesforce reports in a single request."""
    report_ids_raw = payload.get("report_ids") or payload.get("reportIds")
    if not isinstance(report_ids_raw, (list, tuple)):
        raise HTTPException(400, "report_ids must be a list")

    # Normalize and de-duplicate while preserving order
    ordered_ids = []
    seen = set()
    for raw_id in report_ids_raw:
        rid = str(raw_id or "").strip()
        if not rid:
            continue
        if rid in seen:
            continue
        seen.add(rid)
        ordered_ids.append(rid)

    if not ordered_ids:
        raise HTTPException(400, "report_ids must not be empty")

    from ...services.salesforce_gateway import SalesforceGateway
    from ...db import get_session

    account_id = api_ctx.account_id
    if not account_id:
        raise HTTPException(401, "Missing account_id in API context")

    async for db in get_session():
        try:
            gateway = await SalesforceGateway.for_tenant(str(account_id), db)

            tasks = [gateway.export_report(rid) for rid in ordered_ids]
            results = await asyncio.gather(*tasks, return_exceptions=True)

            normalized: List[Dict[str, Any]] = []
            for rid, result in zip(ordered_ids, results):
                if isinstance(result, Exception):
                    log.error("Failed to export report %s in batch: %s", rid, result)
                    normalized.append(
                        {
                            "report_id": rid,
                            "success": False,
                            "error": str(result),
                        }
                    )
                    continue

                report_payload = dict(result or {})
                report_payload.setdefault("report_id", rid)
                if not report_payload.get("success", False):
                    normalized.append(
                        {
                            **report_payload,
                            "success": False,
                            "error": report_payload.get("error", "Export failed"),
                        }
                    )
                    continue

                normalized.append(
                    {
                        **report_payload,
                        "success": True,
                    }
                )

            return JSONResponse(
                content={
                    "ok": True,
                    "data": {
                        "results": normalized,
                    },
                }
            )
        except HTTPException:
            raise
        except Exception as e:
            log.error("Failed to export reports batch %s: %s", ordered_ids, e)
            raise HTTPException(500, f"Failed to export reports batch: {str(e)}")
        finally:
            await db.close()


@router.post("/report/export")
async def export_salesforce_report(
    payload: Dict[str, Any],
    api_ctx=Depends(require_api_key),
):
    """
    Export a Salesforce report and return CSV data.

    Body:
    - report_id: Salesforce Report ID (starts with 00O)
    """
    report_id = payload.get("report_id")
    if not report_id:
        raise HTTPException(400, "report_id required")

    from ...db import get_session

    # Get gateway for this tenant
    account_id = api_ctx.account_id
    if not account_id:
        raise HTTPException(401, "Missing account_id in API context")

    # Create DB session
    async for db in get_session():
        try:
            from ...services.salesforce_gateway import SalesforceGateway

            gateway = await SalesforceGateway.for_tenant(str(account_id), db)

            # Call gateway method
            result = await gateway.export_report(report_id)

            if not result.get("success"):
                raise HTTPException(500, result.get("error", "Export failed"))

            return JSONResponse(content={"ok": True, "data": result})
        except HTTPException:
            raise
        except Exception as e:
            log.error(f"Failed to export report {report_id}: {e}")
            raise HTTPException(500, f"Failed to export report: {str(e)}")
        finally:
            await db.close()
