import httpx

import json

import secrets

import hashlib

import asyncio

import time

import csv

import io

import html

import re

import random

import os
import base64

from contextlib import asynccontextmanager

from collections import deque

from datetime import datetime, timedelta, timezone

from urllib.parse import urlencode, urlparse

from typing import Optional, Dict, Any, List, Tuple


import logging
import xml.etree.ElementTree as ET

try:
    import redis.asyncio as redis_async
    from redis.exceptions import LockError
except Exception:  # pragma: no cover - redis is optional in some test envs
    redis_async = None
    LockError = Exception

from ..security.crypto import encrypt_str, decrypt_str

from .. import settings


log = logging.getLogger(__name__)

BULK_POLL_INITIAL_INTERVAL = 2
BULK_POLL_MAX_INTERVAL = 30
BULK_POLL_MAX_DURATION = 600
MAX_RESULT_FETCH_RETRIES = 3
RESULT_FETCH_RETRY_DELAY = 5


class BulkResultFetchError(Exception):
    """Raised when bulk job results cannot be retrieved after retries."""

    def __init__(self, message: str, job_id: Optional[str] = None) -> None:
        super().__init__(message)
        self.job_id = job_id


class BulkJobFailedError(Exception):
    """Raised when a bulk job completes with a Failed state."""

    def __init__(self, message: str, job_id: Optional[str] = None) -> None:
        super().__init__(message)
        self.job_id = job_id


class BulkJobAbortedError(Exception):
    """Raised when a bulk job completes with an Aborted state."""

    def __init__(self, message: str, job_id: Optional[str] = None) -> None:
        super().__init__(message)
        self.job_id = job_id

REPORT_TYPE_ALIASES = {
    "account": "Account",
    "accounts": "Account",
    "contact": "Contact",
    "contacts": "Contact",
    "lead": "Lead",
    "leads": "Lead",
    "opportunity": "Opportunity",
    "opportunities": "Opportunity",
    "campaign": "Campaign",
    "campaigns": "Campaign",
}


def _utcnow_naive() -> datetime:
    """Return current UTC time as timezone-naive datetime for DB storage."""
    return datetime.utcnow()


def _normalize_db_timestamp(value: Optional[datetime]) -> Optional[datetime]:
    """Normalize timestamps read from DB to timezone-naive UTC."""
    if value is None:
        return None
    if isinstance(value, datetime):
        if value.tzinfo is not None:
            return value.astimezone(timezone.utc).replace(tzinfo=None)
        return value
    return value


def _escape_soql_like(value: str) -> str:
    """Escape user-provided text for inclusion in a SOQL LIKE literal."""

    if value is None:
        return ""

    escaped = value.replace("\\", "\\\\")

    escaped = escaped.replace("'", "\\'")

    escaped = escaped.replace("%", "\\%").replace("_", "\\_")

    return escaped


class SalesforceRateLimiter:
    """Rate limiter for Salesforce API calls with proper concurrency control"""

    def __init__(self, max_per_hour=15000, max_concurrency=25):
        self.max_per_hour = max_per_hour

        self.sem = asyncio.Semaphore(max_concurrency)

        self.times = deque(maxlen=max_per_hour)

    @asynccontextmanager
    async def slot(self):
        """Context manager that holds semaphore during the entire request"""

        await self.sem.acquire()

        try:
            now = time.time()

            # Remove old requests outside the hour window

            while self.times and self.times[0] < now - 3600:
                self.times.popleft()

            # If at limit, wait until oldest request expires

            if len(self.times) >= self.max_per_hour:
                sleep_s = 3600 - (now - self.times[0])

                log.warning(f"Rate limit reached; sleeping {sleep_s:.1f}s")

                await asyncio.sleep(sleep_s)

            self.times.append(time.time())

            yield

        finally:
            self.sem.release()


class SalesforceClient:
    """Async Salesforce client with OAuth and rate limiting"""

    _redis_lock_client = None

    def __init__(self, integration: Optional[Any] = None):
        self.integration = integration
        expires_at = _normalize_db_timestamp(self._integration_value("expires_at"))
        if self.integration and expires_at is not None:
            self.integration.expires_at = expires_at

        self.instance_url = self._integration_value("instance_url")
        self.needs_reconnect = False

        # Decrypt token if encryption is enabled, otherwise use plaintext
        encrypted_access_token = self._integration_value("access_token")
        if integration and encrypted_access_token:
            if settings.ENCRYPTION_KEY:
                try:
                    self.access_token = decrypt_str(encrypted_access_token)
                    if not self.access_token:
                        raise ValueError("Decryption returned empty access token")
                except Exception as e:
                    log.error(
                        "Failed to decrypt Salesforce access token. "
                        "Forcing reconnect: %s",
                        e,
                    )
                    self.access_token = None
                    self.needs_reconnect = True
            else:
                # No encryption key configured - tokens stored plaintext
                self.access_token = encrypted_access_token

        else:
            if integration:
                log.error(
                    "SalesforceClient created but integration.access_token is None/empty!"
                )
            else:
                log.error("SalesforceClient created with no integration!")
            self.access_token = None

        self.rate_limiter = SalesforceRateLimiter()

        self._refresh_lock = None  # Create lazily to avoid event loop issues

    def _integration_dict(self) -> Dict[str, Any]:
        if not self.integration:
            return {}
        data = getattr(self.integration, "__dict__", None)
        if isinstance(data, dict):
            return data
        return {}

    def _integration_value(self, key: str) -> Any:
        data = self._integration_dict()
        if key in data:
            return data.get(key)
        if not self.integration:
            return None
        try:
            return getattr(self.integration, key)
        except Exception:
            # Avoid lazy-load failures bubbling out of token management paths.
            return None

    def _integration_config(self) -> Dict[str, Any]:
        raw = self._integration_value("config")
        if isinstance(raw, dict):
            return dict(raw)
        return {}

    def _ensure_lock(self):
        """Lazily create the asyncio lock when needed"""

        if self._refresh_lock is None:
            self._refresh_lock = asyncio.Lock()

    @classmethod
    def _get_refresh_redis(cls):
        """Return shared Redis client for distributed refresh locks."""
        if redis_async is None:
            return None
        redis_url = getattr(settings, "REDIS_URL", None)
        if not redis_url:
            return None
        client = getattr(cls, "_redis_lock_client", None)
        if client is False:
            return None
        if client is None:
            try:
                cls._redis_lock_client = redis_async.from_url(
                    redis_url, decode_responses=False
                )
            except Exception as exc:  # pragma: no cover - network env dependent
                log.warning(
                    "Distributed refresh locking disabled; Redis unavailable: %s", exc
                )
                cls._redis_lock_client = False
        client = getattr(cls, "_redis_lock_client", None)
        return client if client not in (None, False) else None

    async def _acquire_distributed_refresh_lock(self):
        """Use Redis to ensure only one worker refreshes a token at a time."""
        redis_client = self._get_refresh_redis()
        if not redis_client or not self.integration:
            return None
        lock_id = getattr(self.integration, "id", None) or getattr(
            self.integration, "account_id", None
        )
        if not lock_id:
            return None
        lock_name = f"sf:refresh:{lock_id}"
        lock = redis_client.lock(lock_name, timeout=120)
        try:
            acquired = await lock.acquire(blocking=True, blocking_timeout=30)
            if acquired:
                return lock
        except Exception as exc:  # pragma: no cover - network env dependent
            log.warning(
                "Could not acquire distributed refresh lock for %s: %s", lock_name, exc
            )
        return None

    async def _release_distributed_refresh_lock(self, lock):
        if not lock:
            return
        try:
            await lock.release()
        except LockError:
            return
        except Exception as exc:  # pragma: no cover
            log.warning("Error releasing distributed refresh lock: %s", exc)

    async def _request_with_backoff(
        self, method: str, url: str, **kwargs
    ) -> httpx.Response:
        """Make HTTP request with exponential backoff for 429/503 errors"""

        # Configure timeout: 30s total, 15s for connection/TLS handshake

        timeout = httpx.Timeout(
            timeout=30.0,  # Total timeout
            connect=15.0,  # Connection/TLS handshake timeout
        )

        # Override timeout if provided in kwargs

        if "timeout" in kwargs:
            timeout = kwargs.pop("timeout")

        # Check for proxy configuration

        proxy = None

        if os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy"):
            proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy")

            log.debug(f"Using proxy: {proxy}")

        delay = 1.0

        for attempt in range(6):  # ~1+2+4+8+16+32s
            async with self.rate_limiter.slot():
                try:
                    # Configure client with proxy if needed

                    client_kwargs = {"timeout": timeout, "verify": True}

                    if proxy:
                        client_kwargs["proxies"] = {"https://": proxy}

                    async with httpx.AsyncClient(**client_kwargs) as client:
                        r = await getattr(client, method)(url, **kwargs)

                except httpx.ConnectTimeout as e:
                    # Special handling for TLS handshake timeouts

                    if attempt < 5:  # Retry on timeout
                        wait_time = min(delay * (attempt + 1), 10)  # Cap at 10 seconds

                        log.warning(
                            f"TLS handshake timeout (attempt {attempt+1}/6), retrying in {wait_time:.1f}s: {e}"
                        )

                        await asyncio.sleep(wait_time)

                        continue

                    else:
                        log.error(f"TLS handshake failed after 6 attempts: {e}")

                        # Provide helpful error message

                        raise httpx.ConnectTimeout(
                            f"Unable to connect to Salesforce. This may be due to:\n"
                            f"1. Network connectivity issues\n"
                            f"2. Firewall blocking HTTPS connections\n"
                            f"3. Salesforce service temporarily unavailable\n"
                            f"4. Corporate proxy configuration needed\n"
                            f"Original error: {e}"
                        ) from e

                except httpx.ReadTimeout as e:
                    # Handle read timeouts separately

                    if attempt < 5:
                        wait_time = min(delay * (attempt + 1), 10)

                        log.warning(
                            f"Read timeout (attempt {attempt+1}/6), retrying in {wait_time:.1f}s: {e}"
                        )

                        await asyncio.sleep(wait_time)

                        continue

                    else:
                        log.error(f"Read timeout after 6 attempts: {e}")

                        raise

            if r.status_code in (429, 503):
                retry_after = r.headers.get("Retry-After")

                sleep_s = float(retry_after) if retry_after else delay

                log.warning(
                    f"{r.status_code} from SFDC, retrying in {sleep_s:.1f}s (attempt {attempt+1})"
                )

                await asyncio.sleep(sleep_s)

                delay *= 2

                continue

            # Don't raise for 401 - let caller handle token refresh

            if r.status_code == 401:
                return r

            # Map common SFDC errors to friendly messages

            if r.status_code >= 400:
                try:
                    error_data = r.json()

                    # Log error BEFORE raising
                    log.error(
                        f"Salesforce API error (status={r.status_code}, url={url}): {json.dumps(error_data, indent=2)}"
                    )

                    error_code = (
                        error_data[0].get("errorCode")
                        if isinstance(error_data, list)
                        else None
                    )

                    if error_code == "INVALID_FIELD":
                        raise ValueError(
                            f"Invalid field in query: {error_data[0].get('message')}"
                        )

                    elif error_code == "MALFORMED_QUERY":
                        raise ValueError(
                            f"Malformed SOQL query: {error_data[0].get('message')}"
                        )

                    elif error_code == "INSUFFICIENT_ACCESS":
                        raise PermissionError(
                            f"Insufficient access: {error_data[0].get('message')}"
                        )

                except (json.JSONDecodeError, KeyError, IndexError):
                    # Log text error if JSON parsing fails
                    log.error(
                        f"Salesforce API error (status={r.status_code}, url={url}): {r.text}"
                    )

            r.raise_for_status()

            return r

        raise RuntimeError("Exceeded retries after 429/503")

    async def calibrate_limits(self, db_callback=None):
        """Dynamically adjust rate limits based on org limits"""

        if not await self._ensure_valid_token(db_callback):
            return

        # Use proper timeout for API limit check

        timeout = httpx.Timeout(timeout=30.0, connect=15.0)

        async with httpx.AsyncClient(timeout=timeout, verify=True) as client:
            r = await client.get(
                f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/limits",
                headers={"Authorization": f"Bearer {self.access_token}"},
            )

            r.raise_for_status()

            limits = r.json()

        daily = limits.get("DailyApiRequests", {})

        if daily and daily.get("Max"):
            # Use 80% of hourly average to leave headroom

            per_hour = max(int(daily["Max"] / 24 * 0.8), 200)

            log.info(
                f"Calibrating rate limit to {per_hour} requests/hour based on org limits"
            )

            self.rate_limiter = SalesforceRateLimiter(
                max_per_hour=per_hour, max_concurrency=25
            )

    @staticmethod
    async def get_authorization_url(
        redis_conn, account_id: str, scopes: List[str] = ["api", "refresh_token"]
    ) -> Tuple[str, str]:
        """Generate OAuth authorization URL with CSRF protection"""

        # Generate secure random state

        state = secrets.token_urlsafe(32)

        state_hash = hashlib.sha256(state.encode()).hexdigest()

        # Store state in Redis with account_id for verification (10 min expiry)

        if redis_conn:
            await redis_conn.set(f"oauth_state:{state_hash}", str(account_id), ex=600)

        params = {
            "response_type": "code",
            "client_id": settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID,
            "redirect_uri": settings.SALESFORCE_REDIRECT_URI
            or settings.SF_REDIRECT_URI,
            "scope": " ".join(scopes),
            "state": state,
            "prompt": "consent",  # Ensures refresh_token is returned
        }

        # Support sandbox environments

        auth_host = getattr(settings, "SF_AUTH_HOST", "https://login.salesforce.com")

        auth_url = f"{auth_host}/services/oauth2/authorize?{urlencode(params)}"

        return auth_url, state

    @staticmethod
    async def verify_state(redis_conn, state: str, account_id: str) -> bool:
        """Verify OAuth state to prevent CSRF"""

        if not state or not redis_conn:
            return False

        state_hash = hashlib.sha256(state.encode()).hexdigest()

        stored_account = await redis_conn.get(f"oauth_state:{state_hash}")

        if stored_account:
            # Delete state after verification (one-time use)

            await redis_conn.delete(f"oauth_state:{state_hash}")

            return stored_account.decode() == str(account_id)

        return False

    async def exchange_code_for_tokens(self, code: str) -> Dict[str, Any]:
        """Exchange authorization code for tokens"""

        # Support sandbox environments

        auth_host = getattr(settings, "SF_AUTH_HOST", "https://login.salesforce.com")

        # Use proper timeout for OAuth operations

        timeout = httpx.Timeout(timeout=30.0, connect=15.0)

        async with httpx.AsyncClient(timeout=timeout, verify=True) as client:
            response = await client.post(
                f"{auth_host}/services/oauth2/token",
                data={
                    "grant_type": "authorization_code",
                    "code": code,
                    "client_id": settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID,
                    "client_secret": settings.SALESFORCE_CLIENT_SECRET
                    or settings.SF_CLIENT_SECRET,
                    "redirect_uri": settings.SALESFORCE_REDIRECT_URI
                    or settings.SF_REDIRECT_URI,
                },
            )

            response.raise_for_status()

            return response.json()

    async def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
        """Refresh access token using refresh token"""

        # Support sandbox environments

        auth_host = getattr(settings, "SF_AUTH_HOST", "https://login.salesforce.com")

        # Use proper timeout for OAuth operations

        timeout = httpx.Timeout(timeout=30.0, connect=15.0)

        async with httpx.AsyncClient(timeout=timeout, verify=True) as client:
            response = await client.post(
                f"{auth_host}/services/oauth2/token",
                data={
                    "grant_type": "refresh_token",
                    "refresh_token": refresh_token,
                    "client_id": settings.SALESFORCE_CLIENT_ID or settings.SF_CLIENT_ID,
                    "client_secret": settings.SALESFORCE_CLIENT_SECRET
                    or settings.SF_CLIENT_SECRET,
                },
            )

            response.raise_for_status()

            return response.json()

    def _token_expiring_soon(self) -> bool:
        """Check if token is expiring within 5 minutes"""

        if not self.integration:
            return False

        expires_at = _normalize_db_timestamp(self._integration_value("expires_at"))
        if not expires_at:
            return False
        self.integration.expires_at = expires_at

        time_until_expiry = (expires_at - _utcnow_naive()).total_seconds()

        return time_until_expiry < 300  # Less than 5 minutes

    async def _refresh_if_needed(self, db_callback=None) -> bool:
        """Refresh token if needed with lock to prevent double refresh



        Returns:

            True if token is valid (refreshed or still valid)

            False if refresh failed or reconnect needed

        """

        if not self.integration:
            return False

        if self.needs_reconnect:
            raise ValueError(
                "Salesforce authentication failed - please reconnect your account"
            )

        if not self._token_expiring_soon():
            return self.access_token is not None

        self._ensure_lock()

        async with self._refresh_lock:
            # Check again after acquiring lock - another waiter may have refreshed

            if not self._token_expiring_soon():
                return self.access_token is not None

            distributed_lock = await self._acquire_distributed_refresh_lock()
            if distributed_lock is None:
                log.warning(
                    "Proceeding without distributed refresh lock for integration %s",
                    getattr(self.integration, "id", "unknown"),
                )

            try:
                # Another worker may have refreshed while we were waiting to acquire Redis lock
                if not self._token_expiring_soon():
                    return self.access_token is not None

                log.info("Token expiring soon, refreshing proactively")

                encrypted_refresh_token = self._integration_value("refresh_token")
                refresh_token = (
                    decrypt_str(encrypted_refresh_token)
                    if settings.ENCRYPTION_KEY
                    else encrypted_refresh_token
                )

                if not refresh_token:
                    self.needs_reconnect = True

                    raise ValueError(
                        "No refresh token available - please reconnect Salesforce"
                    )

                tokens = await self.refresh_access_token(refresh_token)

                # Check for error response

                if "error" in tokens:
                    if tokens["error"] == "invalid_grant":
                        self.needs_reconnect = True

                        # Persist to database

                        if db_callback and self.integration:
                            cfg = self._integration_config()

                            cfg.update(
                                {
                                    "needs_reconnect": True,
                                    "reconnect_reason": "token_revoked_or_expired",
                                    "reconnect_message": "Salesforce authentication failed - please reconnect",
                                }
                            )

                            self.integration.config = cfg

                            # Optionally mark as disconnected

                            # self.integration.status = IntegrationStatus.DISCONNECTED

                            await db_callback(self.integration)

                        raise ValueError(
                            "Salesforce refresh token expired or revoked - please reconnect"
                        )

                    else:
                        raise ValueError(
                            f"Token refresh failed: {tokens.get('error_description', tokens['error'])}"
                        )

                # Update tokens

                self.access_token = tokens["access_token"]

                self.integration.access_token = (
                    encrypt_str(tokens["access_token"])
                    if settings.ENCRYPTION_KEY
                    else tokens["access_token"]
                )

                # Salesforce sometimes omits refresh_token in response (means keep existing)

                if "refresh_token" in tokens:
                    self.integration.refresh_token = (
                        encrypt_str(tokens["refresh_token"])
                        if settings.ENCRYPTION_KEY
                        else tokens["refresh_token"]
                    )

                    log.info("New refresh_token received and saved during refresh")

                else:
                    log.info("No new refresh_token in response, keeping existing one")

                self.integration.expires_at = _utcnow_naive() + timedelta(
                    seconds=int(tokens.get("expires_in", 7200))
                )

                # Clear any reconnect flags on successful refresh

                self.needs_reconnect = False

                if self.integration:
                    cfg = self._integration_config()

                    for k in (
                        "needs_reconnect",
                        "reconnect_reason",
                        "reconnect_message",
                    ):
                        cfg.pop(k, None)

                    self.integration.config = cfg

                # Persist immediately if callback provided

                if db_callback:
                    await db_callback(self.integration)

                    log.info(
                        "Token refreshed and persisted successfully, cleared reconnect flags"
                    )

                else:
                    log.warning(
                        "Token refreshed but no db_callback provided to persist"
                    )

                return True

            except httpx.HTTPStatusError as e:
                if e.response.status_code == 400:
                    # Parse error response

                    try:
                        error_data = e.response.json()

                        if error_data.get("error") == "invalid_grant":
                            self.needs_reconnect = True

                            raise ValueError(
                                "Salesforce refresh token expired or revoked - please reconnect"
                            )

                    except:
                        pass

                log.error(f"Failed to refresh token: {e}")

                return False

            except Exception as e:
                log.error(f"Failed to refresh token: {e}")

                if "please reconnect" in str(e):
                    raise

                return False

            finally:
                await self._release_distributed_refresh_lock(distributed_lock)

    async def _ensure_valid_token(self, db_callback=None) -> bool:
        """Ensure we have a valid token, refresh if needed



        Args:

            db_callback: Optional async function to save the updated integration

        """

        return await self._refresh_if_needed(db_callback)

    async def soql(self, query: str, db_callback=None) -> Dict[str, Any]:
        """Execute SOQL query with rate limiting and auto-refresh"""

        # Ensure valid token

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        response = await self._request_with_backoff(
            "get",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/query",
            params={"q": query},
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            # Try to refresh exactly once and retry

            log.info("Got 401, attempting token refresh and retry")

            try:
                # Force a refresh even if we think token is valid

                self._ensure_lock()

                async with self._refresh_lock:
                    encrypted_refresh_token = self._integration_value("refresh_token")
                    refresh_token = (
                        decrypt_str(encrypted_refresh_token)
                        if settings.ENCRYPTION_KEY
                        else encrypted_refresh_token
                    )

                    if not refresh_token:
                        self.needs_reconnect = True

                        raise ValueError(
                            "Salesforce authentication failed - please reconnect your account"
                        )

                    tokens = await self.refresh_access_token(refresh_token)

                    if "error" in tokens:
                        if tokens["error"] == "invalid_grant":
                            self.needs_reconnect = True

                            # Persist to database

                            if db_callback and self.integration:
                                cfg = self._integration_config()

                                cfg.update(
                                    {
                                        "needs_reconnect": True,
                                        "reconnect_reason": "token_revoked_or_expired",
                                        "reconnect_message": "Salesforce authentication failed - refresh token expired. Please reconnect your account",
                                    }
                                )

                                self.integration.config = cfg

                                await db_callback(self.integration)

                            raise ValueError(
                                "Salesforce authentication failed - refresh token expired. Please reconnect your account"
                            )

                        raise ValueError(
                            f"Token refresh failed: {tokens.get('error_description', tokens['error'])}"
                        )

                    # Update tokens

                    self.access_token = tokens["access_token"]

                    self.integration.access_token = (
                        encrypt_str(tokens["access_token"])
                        if settings.ENCRYPTION_KEY
                        else tokens["access_token"]
                    )

                    if "refresh_token" in tokens:
                        self.integration.refresh_token = (
                            encrypt_str(tokens["refresh_token"])
                            if settings.ENCRYPTION_KEY
                            else tokens["refresh_token"]
                        )

                    self.integration.expires_at = _utcnow_naive() + timedelta(
                        seconds=int(tokens.get("expires_in", 7200))
                    )

                    # Clear any reconnect flags on successful refresh

                    self.needs_reconnect = False

                    if self.integration:
                        cfg = self._integration_config()

                        for k in (
                            "needs_reconnect",
                            "reconnect_reason",
                            "reconnect_message",
                        ):
                            cfg.pop(k, None)

                        self.integration.config = cfg

                    # Persist immediately

                    if db_callback:
                        await db_callback(self.integration)

                        log.info(
                            "Token refreshed and persisted after 401, cleared reconnect flags"
                        )

                # Retry with new token

                response = await self._request_with_backoff(
                    "get",
                    f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/query",
                    params={"q": query},
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )

                if response.status_code == 401:
                    # Still 401 after refresh - token is bad

                    self.needs_reconnect = True

                    # Persist to database

                    if db_callback and self.integration:
                        cfg = self._integration_config()

                        cfg.update(
                            {
                                "needs_reconnect": True,
                                "reconnect_reason": "auth_failed_after_refresh",
                                "reconnect_message": "Salesforce authentication failed after refresh. Please reconnect your account",
                            }
                        )

                        self.integration.config = cfg

                        await db_callback(self.integration)

                    raise ValueError(
                        "Salesforce authentication failed after refresh. Please reconnect your account"
                    )

            except Exception as e:
                if "please reconnect" in str(e).lower():
                    raise

                log.error(f"Failed to recover from 401: {e}")

                raise ValueError(f"Salesforce authentication failed: {str(e)}")

        return response.json()

    async def tooling_query(self, query: str, db_callback=None) -> Dict[str, Any]:
        """Execute Tooling API SOQL query with rate limiting and auto-refresh"""

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        url = f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/tooling/query"
        response = await self._request_with_backoff(
            "get",
            url,
            params={"q": query},
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            if self.integration:
                self.integration.expires_at = _utcnow_naive()
            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "get",
                    url,
                    params={"q": query},
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )
                if response.status_code == 401:
                    raise ValueError(
                        "Authentication failed - please reconnect Salesforce"
                    )

        response.raise_for_status()
        return response.json()

    def _is_sandbox_instance(self) -> bool:
        """Detect if this is a sandbox or scratch org based on instance URL."""
        if not self.instance_url:
            return False
        url_lower = self.instance_url.lower()
        # Sandbox patterns: *.sandbox.*, --*.scratch.*, test.salesforce.com
        return (
            ".sandbox." in url_lower
            or "--" in url_lower  # Scratch org pattern
            or "test.salesforce.com" in url_lower
            or ".scratch." in url_lower
            or ".cs" in url_lower  # Classic sandbox servers
        )

    async def deploy_metadata(
        self,
        zip_bytes: bytes,
        db_callback=None,
        *,
        check_only: bool = False,
        test_level: str | None = None,
    ) -> Dict[str, Any]:
        """Deploy metadata zip via SOAP Metadata API.

        For sandboxes: Uses NoTestRun (fast, no tests)
        For production: Uses RunLocalTests (required, runs all local tests)

        Note: Production deploys may take several minutes if the org has many
        test classes. Flaky tests in the org can cause deploy failures.
        """

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        # Auto-detect test level based on org type
        if test_level is None:
            if self._is_sandbox_instance():
                test_level = "NoTestRun"
            else:
                test_level = "RunLocalTests"

        api_version = (settings.SF_API_VERSION or "v60.0").lstrip("vV")
        url = f"{self.instance_url}/services/Soap/m/{api_version}"
        zip_b64 = base64.b64encode(zip_bytes).decode("utf-8")

        envelope = f"""<?xml version="1.0" encoding="UTF-8"?>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:met="http://soap.sforce.com/2006/04/metadata">
  <soapenv:Header>
    <met:SessionHeader>
      <met:sessionId>{self.access_token}</met:sessionId>
    </met:SessionHeader>
  </soapenv:Header>
  <soapenv:Body>
    <met:deploy>
      <met:ZipFile>{zip_b64}</met:ZipFile>
      <met:DeployOptions>
        <met:allowMissingFiles>false</met:allowMissingFiles>
        <met:autoUpdatePackage>false</met:autoUpdatePackage>
        <met:checkOnly>{str(bool(check_only)).lower()}</met:checkOnly>
        <met:ignoreWarnings>false</met:ignoreWarnings>
        <met:performRetrieve>false</met:performRetrieve>
        <met:purgeOnDelete>false</met:purgeOnDelete>
        <met:rollbackOnError>true</met:rollbackOnError>
        <met:singlePackage>true</met:singlePackage>
        <met:testLevel>{test_level}</met:testLevel>
      </met:DeployOptions>
    </met:deploy>
  </soapenv:Body>
</soapenv:Envelope>"""

        response = await self._request_with_backoff(
            "post",
            url,
            data=envelope,
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "text/xml",
                "SOAPAction": "deploy",
            },
        )

        if response.status_code == 401:
            if self.integration:
                self.integration.expires_at = _utcnow_naive()
            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "post",
                    url,
                    data=envelope,
                    headers={
                        "Authorization": f"Bearer {self.access_token}",
                        "Content-Type": "text/xml",
                        "SOAPAction": "deploy",
                    },
                )
                if response.status_code == 401:
                    raise ValueError(
                        "Authentication failed - please reconnect Salesforce"
                    )

        response.raise_for_status()
        return self._parse_metadata_deploy_response(response.text)

    @staticmethod
    def _parse_metadata_deploy_response(xml_text: str) -> Dict[str, Any]:
        try:
            root = ET.fromstring(xml_text)
        except Exception as exc:
            raise ValueError(f"Invalid SOAP response: {exc}")

        ns = {
            "soapenv": "http://schemas.xmlsoap.org/soap/envelope/",
            "m": "http://soap.sforce.com/2006/04/metadata",
        }

        fault = root.find(".//soapenv:Fault", ns)
        if fault is not None:
            fault_string = fault.findtext("faultstring") or "Metadata deploy failed"
            raise ValueError(fault_string)

        deploy_id = root.findtext(".//m:result/m:id", default="", namespaces=ns)
        status = root.findtext(".//m:result/m:status", default="", namespaces=ns)
        return {
            "id": deploy_id or None,
            "status": status or "pending",
        }

    async def check_deploy_status(
        self, deployment_id: str, db_callback=None
    ) -> Dict[str, Any]:
        """Check status of an async Metadata API deployment."""
        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        api_version = (settings.SF_API_VERSION or "v60.0").lstrip("vV")
        url = f"{self.instance_url}/services/Soap/m/{api_version}"

        envelope = f"""<?xml version="1.0" encoding="UTF-8"?>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:met="http://soap.sforce.com/2006/04/metadata">
  <soapenv:Header>
    <met:SessionHeader>
      <met:sessionId>{self.access_token}</met:sessionId>
    </met:SessionHeader>
  </soapenv:Header>
  <soapenv:Body>
    <met:checkDeployStatus>
      <met:asyncProcessId>{deployment_id}</met:asyncProcessId>
      <met:includeDetails>true</met:includeDetails>
    </met:checkDeployStatus>
  </soapenv:Body>
</soapenv:Envelope>"""

        response = await self._request_with_backoff(
            "post",
            url,
            data=envelope,
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "text/xml",
                "SOAPAction": "checkDeployStatus",
            },
        )
        response.raise_for_status()
        return self._parse_check_deploy_status_response(response.text)

    @staticmethod
    def _parse_check_deploy_status_response(xml_text: str) -> Dict[str, Any]:
        """Parse checkDeployStatus SOAP response."""
        try:
            root = ET.fromstring(xml_text)
        except Exception as exc:
            raise ValueError(f"Invalid SOAP response: {exc}")

        ns = {
            "soapenv": "http://schemas.xmlsoap.org/soap/envelope/",
            "m": "http://soap.sforce.com/2006/04/metadata",
        }

        fault = root.find(".//soapenv:Fault", ns)
        if fault is not None:
            fault_string = fault.findtext("faultstring") or "Check deploy status failed"
            raise ValueError(fault_string)

        result = root.find(".//m:result", ns)
        if result is None:
            return {"done": False, "status": "Unknown", "success": False}

        done = result.findtext("m:done", default="false", namespaces=ns) == "true"
        status = result.findtext("m:status", default="Unknown", namespaces=ns)
        success = result.findtext("m:success", default="false", namespaces=ns) == "true"

        # Extract error messages if any
        errors = []
        for detail in result.findall(".//m:componentFailures", ns):
            problem = detail.findtext("m:problem", default="", namespaces=ns)
            component = detail.findtext("m:fullName", default="", namespaces=ns)
            if problem:
                errors.append(f"{component}: {problem}" if component else problem)

        # Also check for general errors
        error_msg = result.findtext("m:errorMessage", default="", namespaces=ns)
        if error_msg:
            errors.append(error_msg)

        return {
            "done": done,
            "status": status,
            "success": success,
            "errors": errors,
            "error_message": "; ".join(errors) if errors else None,
        }

    async def ui_api_picklist_values_by_record_type(
        self, object_api_name: str, record_type_id: str, db_callback=None
    ) -> Dict[str, Any]:
        """Fetch UI API picklist values for a specific record type."""

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        url = (
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/ui-api/"
            f"object-info/{object_api_name}/picklist-values/{record_type_id}"
        )
        response = await self._request_with_backoff(
            "get",
            url,
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            if self.integration:
                self.integration.expires_at = _utcnow_naive()
            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "get",
                    url,
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )
                if response.status_code == 401:
                    raise ValueError(
                        "Authentication failed - please reconnect Salesforce"
                    )

        response.raise_for_status()
        return response.json()

    async def paginate_soql(
        self,
        query: str,
        db_callback=None,
        rate_limit_callback=None,
        track_callback=None,
    ):
        """Execute SOQL query with pagination support that follows nextRecordsUrl"""

        import asyncio


        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        url = f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/query"

        params = {"q": query}

        # Configure timeout and retries

        HTTP_TIMEOUT = httpx.Timeout(60.0, connect=15.0)

        MAX_ATTEMPTS = 3

        async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client:
            while True:
                for attempt in range(1, MAX_ATTEMPTS + 1):
                    try:
                        if rate_limit_callback:
                            await rate_limit_callback()
                        async with self.rate_limiter.slot():
                            r = await client.get(
                                url,
                                params=params,
                                headers={
                                    "Authorization": f"Bearer {self.access_token}"
                                },
                            )

                        if r.status_code == 401 and await self._ensure_valid_token(
                            db_callback
                        ):
                            continue

                        if r.status_code in (429, 503):
                            retry_after = r.headers.get("Retry-After")
                            wait_time = (
                                float(retry_after)
                                if retry_after
                                else 0.8 * attempt + random.random() * 0.4
                            )
                            log.warning(
                                "SOQL query rate limited (status %s, attempt %s/%s), "
                                "retrying in %.1fs",
                                r.status_code,
                                attempt,
                                MAX_ATTEMPTS,
                                wait_time,
                            )
                            await asyncio.sleep(wait_time)
                            continue

                        if r.status_code == 403:
                            try:
                                body = r.json()
                            except Exception:
                                body = None
                            if (
                                isinstance(body, list)
                                and body
                                and isinstance(body[0], dict)
                                and "REQUEST_LIMIT_EXCEEDED"
                                in str(body[0].get("errorCode", ""))
                            ):
                                wait_time = 0.8 * attempt + random.random() * 0.4
                                log.warning(
                                    "SOQL query request limit exceeded (attempt %s/%s), "
                                    "retrying in %.1fs",
                                    attempt,
                                    MAX_ATTEMPTS,
                                    wait_time,
                                )
                                await asyncio.sleep(wait_time)
                                continue

                        r.raise_for_status()

                        data = r.json()
                        if track_callback:
                            await track_callback(r)

                        # Yield current batch of records

                        yield data.get("records", [])

                        # Check for more pages

                        next_url = data.get("nextRecordsUrl")

                        if not next_url:
                            return  # All pages processed

                        # Update for next iteration

                        url = f"{self.instance_url}{next_url}"

                        params = None  # nextRecordsUrl already contains query params

                        break  # Success, move to next page

                    except httpx.ReadTimeout:
                        if attempt == MAX_ATTEMPTS:
                            log.error(
                                f"SOQL query timed out after {MAX_ATTEMPTS} attempts"
                            )

                            raise

                        wait_time = (
                            0.8 * attempt + random.random() * 0.4
                        )  # Jittered backoff

                        log.warning(
                            f"SOQL query timeout (attempt {attempt}/{MAX_ATTEMPTS}), retrying in {wait_time:.1f}s"
                        )

                        await asyncio.sleep(wait_time)

    def build_merge_deeplink(self, object_type: str, ids: List[str]) -> str:
        """Build deep link to Salesforce merge UI"""

        if not self.instance_url:
            return ""

        # Use DuplicateRecordSet approach for safety

        base = self.instance_url.rstrip("/")

        return f"{base}/lightning/o/DuplicateRecordSet/home"

    async def update_record(
        self, object_type: str, record_id: str, fields: Dict[str, Any], db_callback=None
    ) -> Dict[str, Any]:
        """Update a Salesforce record with field validation and rate limiting"""

        # Ensure valid token

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        response = await self._request_with_backoff(
            "patch",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}/{record_id}",
            json=fields,
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "application/json",
            },
        )

        if response.status_code == 401:
            # Use same refresh pattern as soql

            log.info("Got 401 on update, attempting token refresh and retry")

            if await self._refresh_if_needed(db_callback):
                response = await self._request_with_backoff(
                    "patch",
                    f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}/{record_id}",
                    json=fields,
                    headers={
                        "Authorization": f"Bearer {self.access_token}",
                        "Content-Type": "application/json",
                    },
                )

                if response.status_code == 401:
                    self.needs_reconnect = True

                    raise ValueError(
                        "Salesforce authentication failed after refresh. Please reconnect your account"
                    )

        # Salesforce returns 204 No Content on successful update

        if response.status_code == 204:
            return {"success": True, "id": record_id}

        return response.json()

    async def create_record(
        self, object_type: str, fields: Dict[str, Any], db_callback=None
    ) -> Dict[str, Any]:
        """Create a Salesforce record with rate limiting and token refresh."""

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        # Log field names only to avoid leaking payload data
        field_keys = list(fields.keys()) if isinstance(fields, dict) else []
        log.info(
            "Creating %s with fields: %s",
            object_type,
            ", ".join(field_keys[:25]),
        )

        response = await self._request_with_backoff(
            "post",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}",
            json=fields,
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "application/json",
            },
        )

        if response.status_code == 401:
            log.info("Got 401 on create, attempting token refresh and retry")

            if await self._refresh_if_needed(db_callback):
                response = await self._request_with_backoff(
                    "post",
                    f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}",
                    json=fields,
                    headers={
                        "Authorization": f"Bearer {self.access_token}",
                        "Content-Type": "application/json",
                    },
                )

                if response.status_code == 401:
                    self.needs_reconnect = True

                    raise ValueError(
                        "Salesforce authentication failed after refresh. Please reconnect your account"
                    )

        # Salesforce returns 201 Created with JSON body { id, success, errors }

        if response.status_code in (200, 201):
            try:
                data = response.json()

            except Exception:
                data = {"success": True}

            return data

        # Log error response for debugging
        try:
            error_data = response.json()
            log.error(
                f"Salesforce API error (status={response.status_code}): {json.dumps(error_data, indent=2)}"
            )
        except Exception:
            log.error(
                f"Salesforce API error (status={response.status_code}): {response.text}"
            )

        return response.json()

    async def composite(
        self, payload: Dict[str, Any], db_callback=None
    ) -> Dict[str, Any]:
        """Execute a REST Composite API request."""

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        response = await self._request_with_backoff(
            "post",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/composite",
            json=payload,
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "application/json",
            },
        )

        if response.status_code == 401:
            log.info("Got 401 on composite, attempting token refresh and retry")
            if await self._refresh_if_needed(db_callback):
                response = await self._request_with_backoff(
                    "post",
                    f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/composite",
                    json=payload,
                    headers={
                        "Authorization": f"Bearer {self.access_token}",
                        "Content-Type": "application/json",
                    },
                )
                if response.status_code == 401:
                    self.needs_reconnect = True
                    raise ValueError(
                        "Salesforce authentication failed after refresh. Please reconnect your account"
                    )

        try:
            data = response.json()
        except Exception:
            data = {}

        if response.status_code < 200 or response.status_code >= 300:
            return {"error": data, "status_code": response.status_code}

        if isinstance(data, list):
            return {"error": data, "status_code": response.status_code}

        return data

    async def delete_record(
        self, object_type: str, record_id: str, db_callback=None
    ) -> Dict[str, Any]:
        """Delete a Salesforce record with token refresh and backoff."""

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        url = f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}/{record_id}"

        response = await self._request_with_backoff(
            "delete",
            url,
            headers={
                "Authorization": f"Bearer {self.access_token}",
            },
        )

        if response.status_code == 401 and await self._ensure_valid_token(db_callback):
            response = await self._request_with_backoff(
                "delete",
                url,
                headers={
                    "Authorization": f"Bearer {self.access_token}",
                },
            )

        # 204 No Content on successful delete

        if response.status_code in (200, 202, 204):
            return {"success": True, "id": record_id}

        try:
            return response.json()

        except Exception:
            return {"success": response.status_code // 100 == 2}

    def _describe_cache_key(self, object_type: str) -> str:
        """Generate cache key for describe results including API version"""

        return f"{self.instance_url}|{settings.SF_API_VERSION}|{object_type}"

    async def describe_global(self, db_callback=None) -> Dict[str, Any]:
        """Describe global metadata (list of sObjects)."""
        cache = getattr(self, "_describe_global_cache", None)
        if cache:
            return cache

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        url = f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects"
        response = await self._request_with_backoff(
            "get",
            url,
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            if self.integration:
                self.integration.expires_at = _utcnow_naive()
            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "get",
                    url,
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )

        response.raise_for_status()
        data = response.json()
        self._describe_global_cache = data
        return data

    async def describe_object(
        self, object_type: str, db_callback=None
    ) -> Dict[str, Any]:
        """Get object metadata including fields with caching"""

        # Check cache first

        key = self._describe_cache_key(object_type)

        if not hasattr(self, "_describe_cache"):
            self._describe_cache = {}

        cached = self._describe_cache.get(key)

        if cached:
            return cached

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        response = await self._request_with_backoff(
            "get",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}/describe",
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            # Try to refresh once and retry

            log.info("Got 401 on describe, attempting token refresh")

            # Force token refresh by setting expiry to past

            if self.integration:
                self.integration.expires_at = _utcnow_naive()

            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "get",
                    f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}/describe",
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )

                # If still 401, authentication is broken

                if response.status_code == 401:
                    raise ValueError(
                        "Authentication failed - please reconnect Salesforce"
                    )

        response.raise_for_status()

        data = response.json()

        # Cache the result

        self._describe_cache[key] = data

        return data

    async def get_record(
        self,
        object_type: str,
        record_id: str,
        fields: List[str] = None,
        db_callback=None,
    ) -> Dict:
        """Get a single record from Salesforce"""

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        url = f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/sobjects/{object_type}/{record_id}"

        params = {}

        if fields:
            params["fields"] = ",".join(fields)

        response = await self._request_with_backoff(
            "get",
            url,
            params=params,
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            # Try to refresh once and retry

            log.info("Got 401, attempting token refresh")

            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "get",
                    url,
                    params=params,
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )

        return response.json()

    async def bulk_create(
        self,
        sobject: str,
        records: List[Dict],
        *,
        job_callback=None,
        db_callback=None,
    ) -> Dict[str, Any]:
        """Use Bulk API 2.0 to insert records."""
        return await self._bulk_ingest(
            "insert",
            sobject,
            records,
            job_callback=job_callback,
            db_callback=db_callback,
        )

    async def bulk_update(
        self,
        sobject: str,
        records: List[Dict],
        *,
        job_callback=None,
        db_callback=None,
    ) -> Dict[str, Any]:
        """Use Bulk API 2.0 to update existing records."""
        return await self._bulk_ingest(
            "update",
            sobject,
            records,
            job_callback=job_callback,
            db_callback=db_callback,
        )

    async def bulk_upsert(
        self,
        sobject: str,
        records: List[Dict],
        external_id_field: str,
        *,
        job_callback=None,
        db_callback=None,
    ) -> Dict[str, Any]:
        """Use Bulk API 2.0 to upsert records."""
        return await self._bulk_ingest(
            "upsert",
            sobject,
            records,
            external_id_field=external_id_field,
            job_callback=job_callback,
            db_callback=db_callback,
        )

    async def _poll_bulk_job_with_backoff(
        self,
        job_id: str,
        *,
        max_duration: int = BULK_POLL_MAX_DURATION,
        db_callback=None,
    ) -> Tuple[str, Optional[dict]]:
        """Poll bulk job with exponential backoff."""
        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        interval = BULK_POLL_INITIAL_INTERVAL
        start = time.time()
        job_info: Optional[dict] = None

        while time.time() - start < max_duration:
            r = await self._request_with_backoff(
                "get",
                f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}",
                headers={"Authorization": f"Bearer {self.access_token}"},
            )
            job_info = r.json()
            state = job_info.get("state")

            if state == "JobComplete":
                return "completed", job_info
            if state == "Failed":
                return "failed", job_info
            if state == "Aborted":
                return "aborted", job_info
            if state not in ("Open", "UploadComplete", "InProgress"):
                log.warning(
                    "[BULK] Unknown job state: %s for job %s", state, job_id
                )

            await asyncio.sleep(interval)
            interval = min(interval * 1.5, BULK_POLL_MAX_INTERVAL)

        return "processing", job_info

    @staticmethod
    def _parse_bulk_csv(text: str) -> List[Dict[str, Any]]:
        if not text or not text.strip():
            return []
        return list(csv.DictReader(io.StringIO(text)))

    async def _fetch_bulk_results(self, job_id: str) -> Dict[str, Any]:
        success_response = await self._request_with_backoff(
            "get",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}/successfulResults",
            headers={"Authorization": f"Bearer {self.access_token}"},
        )
        if success_response.status_code != 200:
            raise RuntimeError(
                f"Failed to retrieve successful results (status={success_response.status_code})"
            )
        successful_results = self._parse_bulk_csv(success_response.text)

        failed_response = await self._request_with_backoff(
            "get",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}/failedResults",
            headers={"Authorization": f"Bearer {self.access_token}"},
        )
        if failed_response.status_code != 200:
            raise RuntimeError(
                f"Failed to retrieve failed results (status={failed_response.status_code})"
            )
        failed_results = self._parse_bulk_csv(failed_response.text)

        unprocessed_response = await self._request_with_backoff(
            "get",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}/unprocessedRecords",
            headers={"Authorization": f"Bearer {self.access_token}"},
        )
        if unprocessed_response.status_code != 200:
            raise RuntimeError(
                f"Failed to retrieve unprocessed results (status={unprocessed_response.status_code})"
            )
        unprocessed_results = self._parse_bulk_csv(unprocessed_response.text)

        return {
            "successfulResults": successful_results,
            "failedResults": failed_results,
            "unprocessedRecords": unprocessed_results,
        }

    async def _fetch_bulk_results_with_retry(self, job_id: str) -> Dict[str, Any]:
        for attempt in range(MAX_RESULT_FETCH_RETRIES):
            try:
                return await self._fetch_bulk_results(job_id)
            except Exception as exc:  # noqa: BLE001
                if attempt == MAX_RESULT_FETCH_RETRIES - 1:
                    raise BulkResultFetchError(
                        f"Failed to fetch bulk results after {MAX_RESULT_FETCH_RETRIES} attempts: {exc}",
                        job_id=job_id,
                    ) from exc
                delay = RESULT_FETCH_RETRY_DELAY * (attempt + 1)
                log.warning(
                    "[BULK] Result fetch attempt %d failed for job %s: %s",
                    attempt + 1,
                    job_id,
                    exc,
                )
                await asyncio.sleep(delay)

    async def _bulk_ingest(
        self,
        operation: str,
        sobject: str,
        records: List[Dict],
        *,
        external_id_field: Optional[str] = None,
        job_callback=None,
        db_callback=None,
    ) -> Dict[str, Any]:
        """
        Shared Bulk API 2.0 ingest helper for insert/update/upsert operations.

        Returns dict with keys:
            - successfulResults
            - failedResults
            - unprocessedRecords
        """
        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not records:
            return {
                "successfulResults": [],
                "failedResults": [],
                "unprocessedRecords": [],
            }

        operation = operation.lower()
        if operation not in {"insert", "update", "upsert"}:
            raise ValueError(f"Unsupported bulk operation: {operation}")

        # Validate required identifiers
        if operation == "update":
            for record in records:
                if "Id" not in record and "id" not in record:
                    raise ValueError(
                        "All records must have an 'Id' field for bulk update"
                    )
                if "Id" not in record and "id" in record:
                    record["Id"] = record.pop("id")
        elif operation == "upsert":
            if not external_id_field:
                raise ValueError("external_id_field required for bulk upsert")
            missing_external = [
                record
                for record in records
                if external_id_field not in record or not record.get(external_id_field)
            ]
            if missing_external:
                raise ValueError(
                    f"All records must include '{external_id_field}' for bulk upsert"
                )

        # Create bulk update job
        job_response = await self._request_with_backoff(
            "post",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest",
            json={
                "operation": operation,
                "object": sobject,
                "contentType": "CSV",
                "lineEnding": "LF",
                **(
                    {"externalIdFieldName": external_id_field}
                    if operation == "upsert"
                    else {}
                ),
            },
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "application/json",
            },
        )

        job = job_response.json()
        job_id = job.get("id")

        if not job_id:
            raise RuntimeError(f"Failed to create bulk update job: {job}")

        if job_callback:
            await job_callback(job_id)

        try:
            # Convert records to CSV
            first_field = None
            if operation == "update":
                first_field = "Id"
            elif operation == "upsert":
                first_field = external_id_field

            csv_data = self._records_to_csv(records, first_field=first_field)

            # Upload CSV data
            upload_response = await self._request_with_backoff(
                "put",
                f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}/batches",
                content=csv_data.encode("utf-8"),
                headers={
                    "Authorization": f"Bearer {self.access_token}",
                    "Content-Type": "text/csv",
                },
            )

            if upload_response.status_code not in (200, 201):
                raise RuntimeError(f"Failed to upload CSV data: {upload_response.text}")

            # Close the job to start processing
            await self._request_with_backoff(
                "patch",
                f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}",
                json={"state": "UploadComplete"},
                headers={
                    "Authorization": f"Bearer {self.access_token}",
                    "Content-Type": "application/json",
                },
            )

            poll_status, job_info = await self._poll_bulk_job_with_backoff(
                job_id, max_duration=BULK_POLL_MAX_DURATION, db_callback=db_callback
            )

            if poll_status == "processing":
                raise TimeoutError(
                    f"Bulk update job timed out after {BULK_POLL_MAX_DURATION} seconds"
                )
            if poll_status == "aborted":
                error_msg = (job_info or {}).get("errorMessage", "Unknown reason")
                raise BulkJobAbortedError(
                    f"Bulk update job aborted: {error_msg}", job_id=job_id
                )
            if poll_status == "failed":
                error_msg = (job_info or {}).get("errorMessage", "Unknown error")
                raise BulkJobFailedError(
                    f"Bulk update job failed: {error_msg}", job_id=job_id
                )

            return await self._fetch_bulk_results_with_retry(job_id)

        except (TimeoutError, BulkJobFailedError, BulkJobAbortedError, BulkResultFetchError):
            # Do not abort on completion or timeout; allow caller to resume polling later.
            raise
        except Exception:
            # Abort the job on error
            try:
                await self._request_with_backoff(
                    "patch",
                    f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/ingest/{job_id}",
                    json={"state": "Aborted"},
                    headers={
                        "Authorization": f"Bearer {self.access_token}",
                        "Content-Type": "application/json",
                    },
                )
            except Exception:
                pass  # Ignore abort errors

            raise

    async def bulk_job_results(
        self,
        job_id: str,
        *,
        timeout_sec: int = BULK_POLL_MAX_DURATION,
        db_callback=None,
    ) -> Dict[str, Any]:
        """Poll and fetch results for an existing Bulk API 2.0 ingest job."""
        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        poll_status, job_info = await self._poll_bulk_job_with_backoff(
            job_id, max_duration=timeout_sec, db_callback=db_callback
        )

        if poll_status == "processing":
            raise TimeoutError("Bulk update job timed out")
        if poll_status == "aborted":
            error_msg = (job_info or {}).get("errorMessage", "Unknown reason")
            raise BulkJobAbortedError(
                f"Bulk update job aborted: {error_msg}", job_id=job_id
            )
        if poll_status == "failed":
            error_msg = (job_info or {}).get("errorMessage", "Unknown error")
            raise BulkJobFailedError(
                f"Bulk update job failed: {error_msg}", job_id=job_id
            )

        return await self._fetch_bulk_results_with_retry(job_id)

    async def bulk_query(
        self, soql: str, object_type: str = "Account", db_callback=None
    ) -> List[Dict]:
        """Use Bulk API 2.0 for large result sets with locator pagination"""
        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        # Create bulk query job
        job_response = await self._request_with_backoff(
            "post",
            f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/jobs/query",
            json={"operation": "query", "query": soql},
            headers={
                "Authorization": f"Bearer {self.access_token}",
                "Content-Type": "application/json",
            },
        )

        job = job_response.json()

    def _records_to_csv(
        self, records: List[Dict], first_field: Optional[str] = None
    ) -> str:
        """Convert list of records to CSV string for Bulk API upload."""
        if not records:
            return ""

        # Get all unique fields across all records
        all_fields = set()
        for record in records:
            all_fields.update(record.keys())

        # Determine column order
        field_list = sorted(all_fields)
        if first_field and first_field in all_fields:
            field_list = [first_field] + [f for f in field_list if f != first_field]

        # Build CSV with consistent LF line endings so it matches the Bulk API job configuration.
        output = io.StringIO()
        writer = csv.DictWriter(
            output,
            fieldnames=field_list,
            extrasaction="ignore",
            lineterminator="\n",
        )
        writer.writeheader()
        writer.writerows(records)

        return output.getvalue()

    async def fetch_next(self, next_url: str, db_callback=None) -> Dict[str, Any]:
        """Fetch a next page using Salesforce nextRecordsUrl.



        Accepts either an absolute URL or a relative path starting with

        /services/data/.... Returns the JSON response with records, done,

        and possibly nextRecordsUrl.

        """

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        # Normalize URL

        url = next_url.strip()

        if not url:
            raise ValueError("Empty next_url")

        instance = urlparse(self.instance_url)
        if not instance.scheme or not instance.netloc:
            raise ValueError("Invalid instance URL")

        # Security: only allow Salesforce query endpoints

        if url.startswith("http://") or url.startswith("https://"):
            parsed = urlparse(url)
            if parsed.username or parsed.password:
                raise ValueError("Invalid next_url credentials")
            if parsed.scheme != instance.scheme or parsed.netloc != instance.netloc:
                raise ValueError("Invalid next_url host")
        else:
            # Must be a Salesforce API path

            if not url.startswith("/services/data/"):
                raise ValueError("Invalid next_url path")

            url = f"{instance.scheme}://{instance.netloc}{url}"

        # Fetch the page

        response = await self._request_with_backoff(
            "get",
            url,
            headers={"Authorization": f"Bearer {self.access_token}"},
        )

        if response.status_code == 401:
            # Attempt refresh

            if await self._ensure_valid_token(db_callback):
                response = await self._request_with_backoff(
                    "get",
                    url,
                    headers={"Authorization": f"Bearer {self.access_token}"},
                )

        response.raise_for_status()

        return response.json()

    async def merge_records(
        self,
        object_type: str,
        master_id: str,
        victim_ids: List[str],
        preserve_fields: Optional[Dict[str, List[str]]] = None,
    ) -> Dict[str, Any]:
        """

        Use Salesforce's native merge to combine duplicates.

        This preserves relationships, audit trail, and data integrity.



        Args:

            object_type: "Account", "Contact", or "Lead"

            master_id: The record to keep

            victim_ids: Records to merge into master (max 2 for Accounts/Contacts, 1 for Leads)

            preserve_fields: Dict of {victim_id: [field_names]} to preserve from victims



        Returns:

            Merge result with success status and details

        """

        if object_type not in ["Account", "Contact", "Lead"]:
            raise ValueError(f"Merge not supported for {object_type}")

        # Salesforce merge limits

        if object_type == "Lead" and len(victim_ids) > 1:
            raise ValueError("Lead merge supports only 1 victim record")

        elif len(victim_ids) > 2:
            raise ValueError(f"{object_type} merge supports max 2 victim records")

        # Build merge request

        merge_request = {
            "masterRecord": {"Id": master_id},
            "victimRecords": [{"Id": vid} for vid in victim_ids],
        }

        # Add field preservation if specified

        if preserve_fields:
            for victim_id, fields in preserve_fields.items():
                if victim_id in victim_ids and fields:
                    # Find the victim record in the request

                    for victim in merge_request["victimRecords"]:
                        if victim["Id"] == victim_id:
                            victim["fieldsToNull"] = []  # Don't null these fields

                            victim["fieldsToPreserve"] = fields

                            break

        try:
            # Use the native merge endpoint

            url = (
                f"{self.instance_url}/services/data/v59.0/sobjects/{object_type}/merge"
            )

            async with self.rate_limiter.slot():
                resp = await self._request_with_backoff(
                    "post",
                    url,
                    json=merge_request,
                    headers={
                        "Authorization": f"Bearer {self.access_token}",
                        "Content-Type": "application/json",
                    },
                )

            if resp.status_code == 204:  # Success with no content
                return {
                    "success": True,
                    "master_id": master_id,
                    "merged_ids": victim_ids,
                    "message": f"Successfully merged {len(victim_ids)} records into {master_id}",
                }

            else:
                return {
                    "success": False,
                    "error": resp.text,
                    "status_code": resp.status_code,
                }

        except Exception as e:
            log.error(f"Merge failed: {e}")

            return {"success": False, "error": str(e)}

    async def get_sample_records(
        self, object_type: str, fields: List[str], n: int = 1000, db_callback=None
    ) -> List[Dict]:
        """

        Get a sample of records from a Salesforce object.



        Args:

            object_type: The Salesforce object type (e.g., 'Account', 'Lead')

            fields: List of field names to retrieve

            n: Maximum number of records to retrieve

            db_callback: Optional database callback for token refresh



        Returns:

            List of dictionaries containing the requested fields

        """

        if not await self._ensure_valid_token(db_callback):
            return []

        # Build SOQL query

        fields_str = ", ".join(fields)

        query = f"SELECT {fields_str} FROM {object_type} LIMIT {n}"

        try:
            # Use the existing soql method

            result = await self.soql(query, db_callback)

            return result.get("records", [])

        except Exception as e:
            log.error(f"Failed to get sample records for {object_type}: {e}")

            return []

    async def list_reports(
        self,
        query: str = "",
        report_type: str = "",
        limit: int = 25,
        page_token: str = "",
        db_callback=None,
    ) -> Dict[str, Any]:
        """List Salesforce reports with search/filtering."""
        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")
        if not self.instance_url:
            raise ValueError("No instance URL")

        limit = max(1, min(int(limit or 25), 200))

        try:
            fields = [
                "Id",
                "Name",
                "DeveloperName",
                "FolderName",
                "LastRunDate",
                "LastModifiedDate",
                "Format",
                "Description",
                "OwnerId",
            ]
            soql = f"SELECT {', '.join(fields)} FROM Report"

            conditions: List[str] = []
            safe_query = (query or "").strip()
            if safe_query:
                escaped_query = _escape_soql_like(safe_query)
                conditions.append(f"Name LIKE '%{escaped_query}%'")

            alias = (report_type or "").strip().lower()
            if alias:
                report_type_value = REPORT_TYPE_ALIASES.get(alias)
                if report_type_value:
                    escaped_type = _escape_soql_like(report_type_value)
                    conditions.append(f"DeveloperName LIKE '{escaped_type}%'")

            if conditions:
                soql += " WHERE " + " AND ".join(conditions)

            soql += f" ORDER BY LastModifiedDate DESC LIMIT {limit}"

            log.info(f"Executing SOQL: {soql}")
            result = await self.soql(soql, db_callback)
            log.info(f"SOQL returned {len(result.get('records', []))} records")
            records = result.get("records", [])

            items = []
            for rec in records:
                items.append(
                    {
                        "id": rec.get("Id"),
                        "name": rec.get("Name"),
                        "folder": rec.get("FolderName"),
                        "type": rec.get("Format", ""),
                        "developer_name": rec.get("DeveloperName", ""),
                        "report_type": rec.get("DeveloperName", ""),
                        "owner_id": rec.get("OwnerId"),
                        "last_run_at": rec.get("LastRunDate"),
                        "last_modified_at": rec.get("LastModifiedDate"),
                        "description": rec.get("Description", ""),
                    }
                )

            return {
                "items": items,
                "next_page_token": "",
            }

        except Exception as e:
            log.error(f"Failed to list reports: {e}")
            return {"items": [], "error": str(e)}

    async def export_report(self, report_id: str, db_callback=None) -> Dict[str, Any]:
        """

        Export a Salesforce report and return the data.



        Args:

            report_id: Salesforce Report ID (starts with 00O)

            db_callback: Optional database callback for token refresh



        Returns:

            Dict with report data including CSV or records

        """

        if not await self._ensure_valid_token(db_callback):
            raise ValueError("Not authenticated")

        if not self.instance_url:
            raise ValueError("No instance URL")

        # Validate report ID format (00O followed by 15 or 18 chars)

        if not report_id or not report_id.startswith("00O"):
            raise ValueError(f"Invalid report ID format: {report_id}")

        try:
            # Use Salesforce Analytics API to run the report

            # GET to execute report and get results

            response = await self._request_with_backoff(
                "get",
                f"{self.instance_url}/services/data/{settings.SF_API_VERSION}/analytics/reports/{report_id}?includeDetails=true",
                headers={"Authorization": f"Bearer {self.access_token}"},
            )

            if not response.is_success:
                raise ValueError(
                    f"Report export failed: {response.status_code} - {response.text}"
                )

            report_data = response.json()

            # Extract report metadata and results

            # Salesforce Analytics API returns complex nested structure

            metadata = report_data.get("reportMetadata", {})
            fact_map = report_data.get("factMap", {})
            extended_metadata = report_data.get("reportExtendedMetadata", {})

            # Detail columns come back as internal keys (e.g. ACCOUNT.NAME).
            detail_columns = metadata.get("detailColumns", []) or []
            column_info_sources = [
                extended_metadata.get("detailColumnInfo", {}) or {},
                metadata.get("columnInfo", {}) or {},
            ]

            detail_column_info = {}
            for source in column_info_sources:
                if not isinstance(source, dict):
                    continue
                for key, info in source.items():
                    if key not in detail_column_info:
                        detail_column_info[key] = info or {}

            def _column_label(column_key: str) -> str:
                info = detail_column_info.get(column_key) or {}
                label = info.get("label") or info.get("displayName") or info.get("name")
                if not label:
                    label = str(column_key or "").replace(".", " ").strip()
                return str(label)

            headers = [_column_label(col) for col in detail_columns] or detail_columns[
                :
            ]

            # Extract rows from fact map. For tabular reports the data lives under T!T.
            rows_data = fact_map.get("T!T", {}).get("rows", []) or []
            if not rows_data:
                # Fall back to the first bucket that has row data (covers matrix/summary reports).
                for bucket in fact_map.values():
                    if isinstance(bucket, dict) and bucket.get("rows"):
                        rows_data = bucket.get("rows", [])
                        if rows_data:
                            break

            def _stringify_cell_value(value):
                if isinstance(value, bool):
                    return "TRUE" if value else "FALSE"
                if isinstance(value, (int, float)):
                    return str(value)
                if isinstance(value, str):
                    return value
                if isinstance(value, dict):
                    for key in (
                        "displayValue",
                        "label",
                        "formattedValue",
                        "value",
                        "amount",
                    ):
                        inner = value.get(key)
                        if inner in (None, "") or inner is value:
                            continue
                        return _stringify_cell_value(inner)
                    try:
                        return json.dumps(value, ensure_ascii=False)
                    except Exception:
                        return str(value)
                if value is None:
                    return ""
                return str(value)

            def _clean_cell(cell):
                if not isinstance(cell, dict):
                    return ""

                text_value = ""
                for key in ("displayValue", "label", "formattedValue", "value"):
                    candidate = cell.get(key)
                    if candidate in (None, ""):
                        continue
                    text_value = _stringify_cell_value(candidate)
                    if text_value not in (None, ""):
                        break

                text_value = html.unescape(str(text_value))
                if "<" in text_value and ">" in text_value:
                    text_value = re.sub(r"<[^>]+>", "", text_value)
                    text_value = html.unescape(text_value)

                return text_value.strip()

            csv_rows = [headers]
            column_count = len(headers) if headers else None

            for row in rows_data:
                data_cells = row.get("dataCells", []) if isinstance(row, dict) else []
                row_values = [_clean_cell(cell) for cell in data_cells]

                if column_count is not None:
                    if len(row_values) < column_count:
                        row_values.extend([""] * (column_count - len(row_values)))
                    elif len(row_values) > column_count:
                        row_values = row_values[:column_count]

                csv_rows.append(row_values)

            output = io.StringIO()
            writer = csv.writer(output)
            writer.writerows(csv_rows)
            csv_content = output.getvalue()

            return {
                "success": True,
                "csv": csv_content,
                "rows": max(0, len(csv_rows) - 1),
                "columns": len(headers),
                "report_name": metadata.get("name", ""),
                "report_id": report_id,
                "detail_columns": detail_columns,
                "detail_column_info": detail_column_info,
                "report_metadata": metadata,
                "report_extended_metadata": extended_metadata,
            }

        except Exception as e:
            log.error(f"Failed to export report {report_id}: {e}")

            return {"success": False, "error": str(e), "report_id": report_id}
