from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks, Request
from fastapi.responses import StreamingResponse, PlainTextResponse
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, update, func, delete
from typing import Optional, List, Dict, Any
from datetime import datetime, timedelta
import datetime as _dt
from pydantic import BaseModel, field_validator
import uuid
import asyncio
import copy
import os
import logging
import json
import re

from ...db import get_session
from ...auth_security import require_role, require_account, require_user
from ...services.reporting_emitter import EventEmitter
from ...services.notification_service import notify_approval_request
from ...model_defs.reporting_models import ChangeSet as ORMChangeSet
from ...services.salesforce_gateway import (
    SalesforceGateway,
    get_salesforce_gateway,
    SalesforceConnectionError,
    SalesforceAuthError,
    SalesforceRateLimitError,
    SalesforceNotConfigured,
)

# Initialize logger for this module
logger = logging.getLogger(__name__)

# Salesforce ID validation
SF_ID_RE = re.compile(r"^[A-Za-z0-9]{15}([A-Za-z0-9]{3})?$")

# Salesforce object key prefixes
KEY_PREFIX = {
    "Lead": {"00Q"},
    "Account": {"001"},
    "Contact": {"003"},
    "Opportunity": {"006"},
    "Case": {"500"},
    "Campaign": {"701"},
    "CampaignMember": {"00v"},
}


def _valid_sf_ids(ids: List[str]) -> List[str]:
    """Validate and de-duplicate Salesforce IDs."""
    seen, good, bad = set(), [], []
    for i in ids or []:
        if i and SF_ID_RE.fullmatch(i) and i not in seen:
            seen.add(i)
            good.append(i)
        else:
            if i:
                bad.append(i)
    if bad:
        logger.warning(
            "Dropping %d invalid SFDC Ids from SOQL IN (...): e.g. %s",
            len(bad),
            bad[:3],
        )
    return good


def _filter_by_prefix(ids: List[str], obj: str) -> List[str]:
    """Filter IDs by Salesforce object key prefix."""
    allowed = KEY_PREFIX.get(obj)
    if not allowed:
        return ids
    keep, drop = [], []
    for i in ids:
        if i and len(i) >= 3:
            (keep if i[:3] in allowed else drop).append(i)
    if drop:
        logger.warning(
            "Dropped %d %s Ids with wrong keyprefix: e.g. %s", len(drop), obj, drop[:3]
        )
    return keep


def sf15_to_18(sf_id15: str) -> str:
    """Convert 15-char Salesforce ID to 18-char with checksum."""
    if len(sf_id15) != 15:
        return sf_id15  # Return unchanged if not 15 chars

    suffix = ""
    for i in range(0, 15, 5):
        block = sf_id15[i : i + 5]
        bits = 0
        for j, c in enumerate(block):
            if "A" <= c <= "Z":
                bits |= 1 << j
        suffix += "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"[bits]
    return sf_id15 + suffix


def _canonicalize_ids(ids: List[str]) -> List[str]:
    """Canonicalize IDs to 18-char format."""
    result = []
    for id_val in ids:
        if id_val and len(id_val) == 15:
            result.append(sf15_to_18(id_val))
        else:
            result.append(id_val)
    return result


# Import auth functions - ALWAYS import the real ones
from ...auth import (
    get_current_account as _real_require_account,
    get_current_user as _real_require_user,
)


if os.getenv("ENV", "").lower() in ["development", "dev"]:
    # Override auth in development mode
    logger.info("L2A: Using development mode auth overrides")

    # Create override functions that ignore all parameters
    async def require_account(bearer=None, api_key=None, db=None):
        """Development override - always return dev account"""
        from ... import settings

        return settings.DEV_ACCOUNT_ID

    async def require_user(account_id=None, bearer=None):
        """Development override - always return dev user"""
        return "dev-user"
else:
    # Production mode - use real auth
    require_account = _real_require_account
    require_user = _real_require_user
# from ...main import get_account_from_api_key_or_header  # Not used with dev mode
# Fix Integration name resolution - use alias to prevent shadowing
from ...models import (
    Integration as IntegrationModel,
    IntegrationProvider,
    IntegrationStatus,
)
from ...models import User, UserRole
from ...model_defs.l2a_models import L2APolicy, L2ASuggestion, L2AManualView, L2ABulkJob
from ...services.l2a_service_intelligent import (
    IntelligentL2AService,
    OverwriteMode,
)  # New compliant version!
from ...services.ensemble_explanations import (
    summarize_evidence,
    compact_explanations,
    build_ensemble_summary,
)
from ...services.explain_utils import classify_match_type
from .schemas import L2ASuggestionsResponse
from ...services.l2a_bulk_jobs import run_bulk_link_job
from ...intelligence.decision_logger import DecisionLogger

# Using module-level logger already defined above

router = APIRouter(prefix="/api/v2/l2a", tags=["l2a"])

# Bulk job controls
APPROVAL_THRESHOLD = int(os.getenv("L2A_BULK_APPROVAL_THRESHOLD", "2000"))
CONCURRENCY_LIMIT = int(os.getenv("L2A_BULK_CONCURRENCY_LIMIT", "1"))


def utcnow():
    return _dt.datetime.utcnow()


# Convenience: dependency factory for role-protected routes
def make_role_dependency(*roles: str):
    async def _dep(
        account_id: str = Depends(require_account),
        user_id: str = Depends(require_user),
        db: AsyncSession = Depends(get_session),
    ):
        return await require_role(*roles, account_id=account_id, user_id=user_id, db=db)

    return _dep


# Pre-baked guards
require_admin_or_manager = make_role_dependency("admin", "manager")


@router.get("/test-no-auth")
async def test_no_auth():
    """Test endpoint with absolutely no auth"""
    logger.info("TEST ENDPOINT CALLED!")
    return {"test": "works", "no_auth": True}


# ============ Helper Functions for Evidence Summaries ============
async def _soql_all(sf, query):
    """Helper to fetch all records from a SOQL query, handling pagination."""
    out = []
    res = await sf.soql(query)
    out.extend(res.get("records", []))

    # Handle pagination using nextRecordsUrl if present
    # Since gateway doesn't expose .next(), we need to extract the path and query again
    while not res.get("done") and res.get("nextRecordsUrl"):
        # Extract the query path from nextRecordsUrl (e.g., "/services/data/v59.0/query/01g...")
        next_url = res.get("nextRecordsUrl")
        if next_url:
            # Execute follow-up query using the locator from nextRecordsUrl
            # For now, just get the initial batch since gateway doesn't support pagination
            # TODO: Add pagination support to gateway if needed for large datasets
            break

    return out


def _coerce_list(text):
    """Safely parse JSON text to list."""
    if not text:
        return []
    try:
        return json.loads(text) if isinstance(text, str) else text
    except Exception:
        return []


def _norm01(x):
    """Normalize a score to 0..1 range."""
    try:
        x = float(x)
        return x / 100.0 if x > 1.0 else max(0.0, min(1.0, x))
    except Exception:
        return 0.0


def _norm_details(details):
    """Normalize field_score_details to ensure all scores are 0..1."""
    out = []
    for d in details or []:
        if isinstance(d, dict):
            nd = dict(d)
            nd["score"] = _norm01(nd.get("score", 0.0))
            # Keep weight numeric
            try:
                nd["weight"] = float(nd.get("weight", 0.0))
            except:
                nd["weight"] = 0.0
            out.append(nd)
    return out


def _normalize_contrib(c):
    """Normalize a single contribution from field_score_details."""
    sf = (
        c.get("src_field")
        or c.get("source_field")
        or c.get("source_column")
        or c.get("source")
        or c.get("src")
        or "?"
    )
    rf = (
        c.get("ref_field")
        or c.get("reference_field")
        or c.get("reference_column")
        or c.get("reference")
        or c.get("target")
        or c.get("dst")
        or "?"
    )
    pair = rf if sf == rf else f"{sf} -> {rf}"
    raw = float(c.get("score", 0) or 0)
    pct = int(round(raw if raw > 1 else raw * 100))
    pct = max(0, min(100, pct))
    return {
        "pair": pair,
        "pct": pct,
        "w": c.get("weight", 1),
        "algo": c.get("algo_used") or c.get("algo") or "",
    }


def _reason_text(details):
    """Build clean reason text from field_score_details."""
    items = sorted(
        [_normalize_contrib(c) for c in details],
        key=lambda x: x["pct"] * x["w"],
        reverse=True,
    )
    if not items:
        return "—"
    top = ", ".join([f'{i["pair"]} ({i["pct"]}%)' for i in items[:3]])
    return f"Matched on: {top}"


def _evidence_summary(details):
    """Compute evidence count and strength from field_score_details."""
    if not details:
        return {"evidence_count": 0, "evidence_strength": 0.0}

    total_weight = 0.0
    weighted_sum = 0.0
    count = 0

    for d in details:
        if isinstance(d, dict):
            # Score is already normalized to 0..1 by _norm_details
            score = float(d.get("score", 0.0))
            weight = float(d.get("weight", 1.0))
            total_weight += weight
            weighted_sum += score * weight
            count += 1

    strength = (weighted_sum / total_weight) if total_weight > 0 else 0.0
    # Ensure strength is in 0..1 range
    strength = max(0.0, min(1.0, strength))
    return {"evidence_count": count, "evidence_strength": strength}


# Dev-only wrapper: accepts X-API-Key/api_key when ALLOW_INSECURE_API_KEY_FALLBACK=true
# Auth now handled by security module


# Request model for L2A run
class L2ARunRequest(BaseModel):
    threshold_override: Optional[float] = None  # 0..1 scale
    b2c_mode: Optional[str] = None  # 'strict' | 'block_only' | 'off'
    b2c_whitelist: Optional[List[str]] = (
        None  # List of domains/emails to treat as business
    )
    mode: Optional[str] = None  # 'incremental' (default) | 'rebuild'
    use_multi_algo: Optional[bool] = (
        None  # Override policy setting for ensemble scoring
    )
    campaign_id: Optional[str] = None  # Salesforce Campaign ID to filter leads
    lead_source: Optional[str] = None  # Filter by LeadSource field
    created_date_from: Optional[str] = (
        None  # Filter leads created after this date (YYYY-MM-DD)
    )
    audit_mode: Optional[str] = "unlinked"  # NEW: "unlinked" | "upgrade"

    @field_validator("threshold_override")
    @classmethod
    def validate_threshold(cls, v):
        if v is not None and not (0 <= v <= 1):
            raise ValueError("threshold_override must be between 0 and 1")
        return v

    @field_validator("b2c_mode")
    @classmethod
    def validate_b2c_mode(cls, v):
        if v is not None and v not in ["strict", "block_only", "off"]:
            raise ValueError("b2c_mode must be one of: strict, block_only, off")
        return v

    @field_validator("audit_mode")
    @classmethod
    def validate_audit_mode(cls, v):
        if v not in ["unlinked", "upgrade"]:
            raise ValueError("audit_mode must be one of: unlinked, upgrade")
        return v

    @field_validator("mode")
    @classmethod
    def validate_mode(cls, v):
        if v is not None and v not in ["incremental", "rebuild"]:
            raise ValueError("mode must be one of: incremental, rebuild")
        return v

    @field_validator("campaign_id")
    @classmethod
    def validate_campaign_id(cls, v):
        if v and not re.fullmatch(r"[a-zA-Z0-9]{15,18}", v):
            raise ValueError("campaign_id must be a 15 or 18-character Salesforce ID")
        return v

    @field_validator("created_date_from")
    @classmethod
    def validate_created_date(cls, v):
        if v:
            try:
                datetime.strptime(v, "%Y-%m-%d")
            except ValueError:
                raise ValueError("created_date_from must be in YYYY-MM-DD format")
        return v


@router.post("/run")
async def run_l2a(
    request: Request,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),  # Use centralized auth
    bg_tasks: BackgroundTasks = None,
):
    """Run L2A discovery and generate suggestions."""
    import os

    logger.info("L2A route executing from file: %s (pid=%s)", __file__, os.getpid())

    # Handle invalid JSON gracefully
    try:
        # Check if there's a body
        body = await request.body()
        if body:
            json_data = await request.json()
            payload = L2ARunRequest(**json_data) if json_data else L2ARunRequest()
        else:
            payload = L2ARunRequest()
    except Exception:
        # If JSON parsing fails, use defaults
        payload = L2ARunRequest()

    # Simple local job tracking function
    _JOB_STORE = {}

    def _update(job_id: str, status: str, progress: float, message: str):
        _JOB_STORE[job_id] = {
            "status": status,
            "progress": progress,
            "message": message,
        }

    job_id = str(uuid.uuid4())
    _update(job_id, "queued", 0.0, "Queued")

    try:
        # Check if we have a Salesforce integration first
        integration_result = await db.execute(
            select(IntegrationModel).where(
                IntegrationModel.account_id == str(account_id),
                IntegrationModel.provider == IntegrationProvider.SALESFORCE,
                IntegrationModel.status == IntegrationStatus.CONNECTED,
            )
        )
        integration = integration_result.scalar_one_or_none()

        if not integration:
            _update(job_id, "completed", 1.0, "No Salesforce integration connected")
            return {
                "job_id": job_id,
                "started": False,
                "reason": "No active Salesforce integration found. Please connect Salesforce first.",
                "status": "completed",
            }

        # Create service first
        svc = IntelligentL2AService(
            db, account_id
        )  # Fully compliant with intelligence!

        # Auto-create policy if missing (ensure_policy handles existing policies too)
        logger.info(f"Calling ensure_policy for account {account_id}")
        policy = await svc.ensure_policy(str(account_id), integration.id)
        logger.info(f"ensure_policy returned policy: {policy.id if policy else 'None'}")
        _update(job_id, "running", 0.05, "Starting L2A discovery")

        # simple progress callback the service can call
        def on_progress(pct: float, msg: str = ""):
            _update(job_id, "running", pct, msg)

        # Try to run the L2A service with better error handling
        try:
            result = await svc.run(
                str(account_id),
                threshold_override=payload.threshold_override,
                progress_cb=on_progress,
                b2c_mode=payload.b2c_mode,
                b2c_whitelist=payload.b2c_whitelist or [],
                mode=payload.mode or "incremental",  # Default to incremental
                use_multi_algo=payload.use_multi_algo,  # Pass ensemble scoring preference
                campaign_id=payload.campaign_id,
                lead_source=payload.lead_source,
                created_date_from=payload.created_date_from,
                audit_mode=payload.audit_mode,  # NEW
            )
            suggestions_count = result.get("suggestions_generated", 0)
            certain_count = result.get("certain_count", 0)
            effective_threshold = result.get("intelligence_config", {}).get(
                "threshold", 0.9
            )
            _update(
                job_id,
                "completed",
                1.0,
                f"Found {suggestions_count} suggestions ({certain_count} CERTAIN) at threshold {effective_threshold:.2f}",
            )

            # Build ensemble summary if enabled
            ensemble_summary = {}
            if os.getenv("ENSEMBLE_SIGNALS_ENABLED", "true").lower() == "true":
                # Get a sample of suggestions to build summary
                sample_suggestions = []
                if suggestions_count > 0:
                    suggestions_res = await db.execute(
                        select(L2ASuggestion)
                        .where(
                            L2ASuggestion.policy_id == policy.id,
                            L2ASuggestion.status == "pending",
                        )
                        .order_by(L2ASuggestion.score.desc())
                        .limit(100)
                    )
                    sample_suggestions = suggestions_res.scalars().all()

                    # Convert to dict format for summary
                    sample_items = []
                    for s in sample_suggestions:
                        item = {"reasons": s.reasons or {}}
                        # Build field score details (fsd) with proper priority
                        fsd = []
                        if (
                            hasattr(s, "field_score_details") and s.field_score_details
                        ):  # 1) true engine details
                            fsd = s.field_score_details
                            item["field_score_details"] = fsd
                        elif s.reasons:  # 2) legacy reasons dict
                            # Parse reasons if it's a string
                            reasons = s.reasons
                            if isinstance(reasons, str):
                                try:
                                    reasons = json.loads(reasons)
                                except:
                                    reasons = {}
                            elif not isinstance(reasons, dict):
                                reasons = {}

                            for field_pair, score in reasons.items():
                                if "->" in field_pair:
                                    src, ref = field_pair.split("->", 1)
                                else:
                                    src, ref = field_pair, field_pair
                                fsd.append(
                                    {
                                        "src_field": src or field_pair,
                                        "ref_field": ref or field_pair,
                                        "score": float(score),
                                        "weight": 1.0,
                                        "algorithm": "",
                                    }
                                )
                        summary = summarize_evidence(
                            fsd, use_multi_algo=bool(payload.use_multi_algo)
                        )
                        item.update(summary)
                        item["match_type"] = classify_match_type(fsd)  # Add match type
                        sample_items.append(item)

                    ensemble_summary = build_ensemble_summary(
                        sample_items,
                        present_only_scoring=True,
                        multi_algo_enabled=bool(payload.use_multi_algo),
                    )

            # Store additional data in job result
            if job_id in _JOB_STORE:
                _JOB_STORE[job_id].update(
                    {
                        "suggestions_count": suggestions_count,
                        "certain_count": certain_count,
                        "ensemble_summary": ensemble_summary,
                    }
                )

            # For synchronous runs, return 200 OK
            response = {
                "job_id": None,  # No background job for sync runs
                "status": "completed",
                "started": False,  # Indicates sync completion
                "mode": "sync",  # Explicit mode indicator
                "suggestions_count": suggestions_count,
                "certain_count": certain_count,
                "effective_threshold": effective_threshold,
            }

            if ensemble_summary:
                response["ensemble_summary"] = ensemble_summary

            return response
        except Exception as e:
            logger.exception("L2A run failed for account %s", account_id)
            _update(job_id, "error", 1.0, f"Error: {e}")
            # Raise HTTPException instead of swallowing the error
            raise HTTPException(status_code=500, detail=f"L2A failed: {e}")
    except ValueError as e:
        import traceback

        error_msg = str(e)
        _update(job_id, "failed", 1.0, f"Error: {error_msg[:200]}")
        print(f"L2A task error: {e}")
        traceback.print_exc()

        # Check for authentication errors and return proper status code
        if (
            "Authentication failed" in error_msg
            or "401" in error_msg
            or "Salesforce authentication" in error_msg
        ):
            raise HTTPException(
                status_code=401,
                detail={
                    "code": "SALESFORCE_AUTH",
                    "message": error_msg,
                    "error": "Salesforce authentication failed. Please reconnect your Salesforce account.",
                },
            )
        # Check for configuration/mapping errors
        elif (
            "Bad mapping" in error_msg
            or "Invalid mapping" in error_msg
            or "field_mappings" in error_msg
        ):
            raise HTTPException(
                status_code=400,
                detail={
                    "code": "INTEL_CONFIG_INVALID",
                    "message": error_msg,
                    "details": {"path": ["field_mappings"]},
                },
            )
        raise HTTPException(status_code=500, detail=error_msg)
    except KeyError as e:
        import traceback

        error_msg = f"Configuration error: {str(e)}"
        _update(job_id, "failed", 1.0, f"Error: {error_msg[:200]}")
        print(f"L2A config error: {e}")
        traceback.print_exc()
        raise HTTPException(
            status_code=400,
            detail={
                "code": "INTEL_CONFIG_INVALID",
                "message": error_msg,
                "details": {"path": [str(e)]},
            },
        )
    except Exception as e:
        import traceback

        _update(job_id, "failed", 1.0, f"Error: {str(e)[:200]}")
        print(f"L2A task error: {e}")
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/campaigns")
async def list_campaigns(
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # Use gateway instead
):
    """Get available Salesforce campaigns for filtering leads."""
    # No longer need to check integration or create client - gateway handles it

    try:
        # Gateway handles token persistence automatically

        # Query active campaigns - use built-in counters only
        query = """
            SELECT Id, Name, Status, StartDate, EndDate, NumberOfLeads
            FROM Campaign
            WHERE IsActive = true
            ORDER BY LastModifiedDate DESC
            LIMIT 100
        """

        result = await sf.soql(query)

        campaigns = []
        for rec in result.get("records", []):
            campaigns.append(
                {
                    "id": rec["Id"],
                    "name": rec["Name"],
                    "status": rec.get("Status"),
                    "start_date": rec.get("StartDate"),
                    "end_date": rec.get("EndDate"),
                    "lead_count": rec.get("NumberOfLeads", 0),
                    "total_members": rec.get("NumberOfLeads", 0),
                }
            )

        return {"campaigns": campaigns, "total": len(campaigns)}

    except (SalesforceNotConfigured, SalesforceAuthError) as e:
        logger.warning(f"Salesforce not connected: {e}")
        raise HTTPException(
            status_code=424,
            detail="Salesforce integration is not connected. Please connect Salesforce first.",
        )
    except SalesforceConnectionError as e:
        logger.error(f"Salesforce connection error: {e}")
        raise HTTPException(
            status_code=503, detail="Salesforce connection temporarily unavailable"
        )
    except Exception as e:
        logger.error(f"Failed to fetch campaigns: {e}")
        return {"campaigns": [], "error": str(e)}


@router.get(
    "/suggestions",
    response_model=L2ASuggestionsResponse,
    response_model_exclude_none=True,
)
async def list_suggestions(
    request: Request,  # Added for query param access
    page: int = 1,
    page_size: int = 50,
    status: Optional[str] = "pending",
    min_score: Optional[float] = Query(None, ge=0.0, le=1.0),
    tier: Optional[str] = None,  # Remove regex validation temporarily
    expand: bool = True,  # hydrate display fields
    sort: Optional[str] = Query(
        None,
        description="Sort order: created_desc|created_asc|score_desc|score_asc (default score_desc)",
    ),
    include: Optional[str] = Query(
        None,
        description="Comma-separated list of additional fields (e.g., 'explanations')",
    ),
    campaign_id: Optional[str] = Query(
        None, description="Filter by campaign membership"
    ),
    lead_source: Optional[str] = Query(None, description="Filter by lead source"),
    created_date_from: Optional[str] = Query(
        None, description="Filter by created date (YYYY-MM-DD)"
    ),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # ADD SALESFORCE GATEWAY
):
    pols = await db.execute(
        select(L2APolicy.id).where(
            L2APolicy.account_id == str(account_id),
            L2APolicy.status == "ACTIVE",
        )
    )
    pol_ids = [r[0] for r in pols.all()]
    if not pol_ids:
        return {"items": [], "total": 0, "page": page, "page_size": page_size}

    base = select(L2ASuggestion).where(L2ASuggestion.policy_id.in_(pol_ids))
    if status and status != "all":
        base = base.where(L2ASuggestion.status == status)
    if min_score is not None:
        base = base.where(L2ASuggestion.score >= min_score)
    if tier:
        base = base.where(L2ASuggestion.tier == tier)

    # Apply population filters if provided
    lead_ids_filter = None
    if campaign_id or lead_source or created_date_from:
        # Gateway is now injected as dependency, no need to check integration
        try:
            # Build Lead query with filters using subselects
            lead_query = "SELECT Id FROM Lead WHERE IsConverted = false"

            if campaign_id:
                # Validate campaign ID format
                if not re.fullmatch(r"[a-zA-Z0-9]{15,18}", campaign_id):
                    raise HTTPException(
                        status_code=400, detail="Invalid campaign ID format"
                    )
                # Use subselect instead of materializing IDs
                lead_query += f" AND Id IN (SELECT LeadId FROM CampaignMember WHERE CampaignId = '{campaign_id}')"

            if lead_source:
                escaped_source = lead_source.replace("\\", "\\\\").replace("'", "\\'")
                lead_query += f" AND LeadSource = '{escaped_source}'"

            if created_date_from:
                try:
                    date_obj = datetime.strptime(created_date_from, "%Y-%m-%d")
                    lead_query += (
                        f" AND CreatedDate >= {date_obj.strftime('%Y-%m-%dT00:00:00Z')}"
                    )
                except ValueError:
                    raise HTTPException(
                        status_code=400, detail="Invalid date format (use YYYY-MM-DD)"
                    )

            # Fetch all matching lead IDs using pagination helper
            lead_result = await _soql_all(sf, lead_query)
            lead_ids_filter = [r["Id"] for r in lead_result]

            # Apply the lead ID filter to the query
            if lead_ids_filter is not None:
                if not lead_ids_filter:
                    # No matching leads, return empty result
                    return {
                        "items": [],
                        "total": 0,
                        "page": page,
                        "page_size": page_size,
                    }
                base = base.where(L2ASuggestion.lead_id.in_(lead_ids_filter))
        except SalesforceConnectionError as e:
            # If Salesforce connection fails, log and continue without filters
            logger.warning(
                f"Unable to apply population filters due to SF connection error: {e}"
            )
        except Exception as e:
            logger.error(f"Error applying population filters: {e}")
            raise HTTPException(
                status_code=500, detail="Error applying population filters"
            )

    total = (
        await db.execute(select(func.count()).select_from(base.subquery()))
    ).scalar_one()
    # Apply sorting
    order_expr = L2ASuggestion.score.desc()
    if sort == "created_desc":
        order_expr = L2ASuggestion.created_at.desc()
    elif sort == "created_asc":
        order_expr = L2ASuggestion.created_at.asc()
    elif sort == "score_asc":
        order_expr = L2ASuggestion.score.asc()

    q = base.order_by(order_expr).offset((page - 1) * page_size).limit(page_size)
    res = await db.execute(q)
    rows = res.scalars().all()

    items = []
    for s in rows:
        # Parse field_score_details and normalize scores to 0..1
        details_raw = _coerce_list(s.field_score_details)
        details = _norm_details(details_raw)  # Normalize all scores to 0..1
        evidence = _evidence_summary(details)  # Uses normalized details

        items.append(
            {
                "id": s.id,
                "lead_id": s.lead_id,
                "account_id": s.account_id,
                "score": s.score,  # Already 0..1
                "tier": s.tier,
                "reasons": s.reasons,
                "explanation": s.explanation,
                "status": s.status,
                "created_at": s.created_at,
                "field_score_details": details,  # Return normalized details
                "evidence_count": evidence["evidence_count"],
                "evidence_strength": evidence["evidence_strength"],  # Guaranteed 0..1
                "reasons_text": _reason_text(details),  # Clean formatted reason
                "match_type": classify_match_type(
                    details
                ),  # Add match type classification
            }
        )

    if expand and items:
        # hydrate display fields from Salesforce in 2 batched queries
        # SF gateway is injected via dependency - no need to create client
        logger.info(
            f"Hydration requested: expand={expand}, items={len(items)}, sf_available={sf is not None}"
        )
        if sf:  # Only proceed if gateway is available

            def _chunks(lst, n):
                for i in range(0, len(lst), n):
                    yield lst[i : i + n]

            lead_ids = list({i["lead_id"] for i in items})
            # Collect ALL account IDs (both suggested and incumbent)
            acct_ids = set()
            for item in items:
                if item.get("account_id"):
                    acct_ids.add(item["account_id"])
                if item.get("incumbent_account_id"):
                    acct_ids.add(item["incumbent_account_id"])
            acct_ids = list(acct_ids)

            # Validate and canonicalize Salesforce IDs before querying
            warnings = []

            # Track original counts for warning messages
            orig_lead_count = len(lead_ids)
            orig_acct_count = len(acct_ids)

            # Validate, canonicalize, and filter by prefix
            lead_ids = _valid_sf_ids(lead_ids)
            lead_ids = _canonicalize_ids(lead_ids)
            lead_ids = _filter_by_prefix(lead_ids, "Lead")

            acct_ids = _valid_sf_ids(acct_ids)
            acct_ids = _canonicalize_ids(acct_ids)
            acct_ids = _filter_by_prefix(acct_ids, "Account")

            # Add warnings if IDs were filtered
            if len(lead_ids) < orig_lead_count:
                warnings.append(
                    f"Filtered {orig_lead_count - len(lead_ids)} invalid Lead IDs"
                )
            if len(acct_ids) < orig_acct_count:
                warnings.append(
                    f"Filtered {orig_acct_count - len(acct_ids)} invalid Account IDs"
                )

            # Guard when everything is invalid
            if (
                not lead_ids
                and not acct_ids
                and (orig_lead_count > 0 or orig_acct_count > 0)
            ):
                return {
                    "items": [],
                    "total": 0,
                    "page": page,
                    "page_size": page_size,
                    "warnings": ["No valid Salesforce IDs to fetch after validation"],
                }

            lead_map, acct_map = {}, {}

            # Gateway handles token persistence automatically

            # Salesforce IN() is happiest <=200 ids at a time
            import httpx

            base_allowlist = {
                "Lead": [
                    "Id",
                    "Company",
                    "Name",
                    "FirstName",
                    "LastName",
                    "Email",
                    "Phone",
                    "MobilePhone",
                    "Title",
                    "Industry",
                ],
                "Account": [
                    "Id",
                    "Name",
                    "Website",
                    "Phone",
                    "Industry",
                    "BillingStreet",
                    "BillingCity",
                    "BillingState",
                    "BillingCountry",
                ],
            }

            for batch in _chunks(lead_ids, 200):
                ids = ",".join([f"'{x}'" for x in batch])
                # Add LIMIT to satisfy security policy
                limit = min(len(batch), 1000)  # Cap at 1000 per security policy
                try:
                    # Only request essential fields - pass plan_allowlist to bypass default security policy
                    r = await sf.soql(
                        (
                            "SELECT Id, Company, Name, FirstName, LastName, Email, "
                            "Phone, MobilePhone, Title, Industry "
                            f"FROM Lead WHERE Id IN ({ids}) LIMIT {limit}"
                        ),
                        plan_allowlist=copy.deepcopy(base_allowlist),
                    )
                    for rec in r.get("records", []):
                        lead_map[rec["Id"]] = rec
                except SalesforceRateLimitError:
                    # Rate limit - skip this batch and continue
                    logger.warning("Rate limit hit for Lead batch %d items", len(batch))
                    continue
                except (SalesforceNotConfigured, SalesforceAuthError) as e:
                    # No connection or auth error - return proper error
                    logger.error(f"Salesforce not available for Lead batch: {e}")
                    raise HTTPException(
                        status_code=424,
                        detail="Salesforce integration is not connected. Please connect Salesforce first.",
                    )
                except httpx.HTTPStatusError as e:
                    # Map 400 to a safe client error with a clear message, not a 500
                    if e.response is not None and e.response.status_code == 400:
                        logger.warning(
                            "SFDC 400 for Lead batch %d items: %s", len(batch), e
                        )
                        continue  # Skip just this chunk, keep serving the rest
                    raise
                except Exception as e:
                    # Catch any other errors (including SecurityViolation) and skip this batch
                    logger.warning(f"Error fetching Lead batch: {e}")
                    continue

            for batch in _chunks(acct_ids, 200):
                ids = ",".join([f"'{x}'" for x in batch])
                # Add LIMIT to satisfy security policy
                limit = min(len(batch), 1000)  # Cap at 1000 per security policy
                try:
                    # Only request essential fields - pass plan_allowlist to bypass default security policy
                    r = await sf.soql(
                        (
                            "SELECT Id, Name, Website, Phone, Industry, BillingStreet, "
                            "BillingCity, BillingState, BillingCountry "
                            f"FROM Account WHERE Id IN ({ids}) LIMIT {limit}"
                        ),
                        plan_allowlist=copy.deepcopy(base_allowlist),
                    )
                    for rec in r.get("records", []):
                        acct_map[rec["Id"]] = rec
                except SalesforceRateLimitError:
                    # Rate limit - skip this batch and continue
                    logger.warning(
                        "Rate limit hit for Account batch %d items", len(batch)
                    )
                    continue
                except (SalesforceNotConfigured, SalesforceAuthError) as e:
                    # No connection or auth error - return proper error
                    logger.error(f"Salesforce not available for Account batch: {e}")
                    raise HTTPException(
                        status_code=424,
                        detail="Salesforce integration is not connected. Please connect Salesforce first.",
                    )
                except httpx.HTTPStatusError as e:
                    # Map 400 to a safe client error with a clear message, not a 500
                    if e.response is not None and e.response.status_code == 400:
                        logger.warning(
                            "SFDC 400 for Account batch %d items: %s", len(batch), e
                        )
                        continue  # Skip just this chunk, keep serving the rest
                    raise
                except Exception as e:
                    # Catch any other errors (including SecurityViolation) and skip this batch
                    logger.warning(f"Error fetching Account batch: {e}")
                    continue

            logger.info(
                f"Hydrating {len(items)} items with {len(lead_map)} leads and {len(acct_map)} accounts"
            )
            for i in items:
                l = lead_map.get(i["lead_id"], {})
                a = acct_map.get(i["account_id"], {})
                lead_company = l.get("Company") or l.get("Name")
                if not lead_company:
                    lead_company = " / ".join(
                        filter(
                            None,
                            [
                                l.get("FirstName", "").strip(),
                                l.get("LastName", "").strip(),
                            ],
                        )
                    )
                i.update(
                    {
                        "lead_company": lead_company or None,
                        "lead_email": l.get("Email"),
                        "lead_phone": l.get("Phone") or l.get("MobilePhone"),
                        "account_name": a.get("Name"),
                        "account_website": a.get("Website"),
                        "account_phone": a.get("Phone"),
                    }
                )
                # Hydrate incumbent account details if present
                if i.get("incumbent_account_id"):
                    inc = acct_map.get(i["incumbent_account_id"], {})
                    i.update(
                        {
                            "incumbent_account_name": inc.get("Name"),
                            "incumbent_account_website": inc.get("Website"),
                            "incumbent_account_phone": inc.get("Phone"),
                        }
                    )
            if items:
                logger.info(
                    "Sample item after hydration: lead_company=%s, account_name=%s",
                    items[0].get("lead_company"),
                    items[0].get("account_name"),
                )

    # Add ensemble signals if enabled
    try:
        ensemble_enabled = (
            os.getenv("ENSEMBLE_SIGNALS_ENABLED", "true").lower() == "true"
        )
    except Exception as e:
        logger.error(f"Error checking ENSEMBLE_SIGNALS_ENABLED: {e}")
        ensemble_enabled = True

    if ensemble_enabled:
        # Get policy to check multi-algo setting
        pol_res = await db.execute(
            select(L2APolicy)
            .where(L2APolicy.id.in_(pol_ids), L2APolicy.status == "ACTIVE")
            .limit(1)
        )
        policy = pol_res.scalar_one_or_none()
    # No active policy yet � continue to discover options

    # Get connected Salesforce integration
    integ_res = await db.execute(
        select(IntegrationModel).where(
            IntegrationModel.account_id == str(account_id),
            IntegrationModel.provider == IntegrationProvider.SALESFORCE,
            IntegrationModel.status == IntegrationStatus.CONNECTED,
        )
    )
    integ = integ_res.scalar_one_or_none()
    # Skip ensemble signals if no Salesforce integration (don't fail the whole request)
    if integ:
        use_multi_algo = bool(policy and getattr(policy, "use_multi_algo", False))

        for item in items:
            # Build field score details (fsd) with proper priority
            fsd = []
            if item.get("field_score_details"):  # 1) true engine details
                fsd = item["field_score_details"]
            elif item.get("reasons"):  # 2) legacy reasons dict
                # Parse reasons if it's a string
                reasons = item["reasons"]
                if isinstance(reasons, str):
                    try:
                        reasons = json.loads(reasons)
                    except:
                        reasons = {}
                elif not isinstance(reasons, dict):
                    reasons = {}

                for field_pair, score in reasons.items():
                    if "->" in field_pair:
                        src, ref = field_pair.split("->", 1)
                    else:
                        src, ref = field_pair, field_pair

                    fsd.append(
                        {
                            "src_field": src or field_pair,
                            "ref_field": ref or field_pair,
                            "score": float(score),
                            "weight": 1.0,
                            "algorithm": "",
                        }
                    )

            # Add Tier-1 summary (always included)
            summary = summarize_evidence(fsd, use_multi_algo=use_multi_algo)
            item.update(summary)
            item["match_type"] = classify_match_type(fsd)  # Add match type

            # Tier-2: detailed explanations only when requested
            if include and "explanations" in include:
                try:
                    item["field_score_details"] = (
                        fsd  # Include the actual field score details
                    )
                    item["explanations"] = compact_explanations(fsd)
                except Exception as e:
                    logger.error(f"Error generating explanations: {e}, fsd={fsd}")
                    item["explanations"] = []  # Return empty array on error

    # Return the payload directly - FastAPI will handle validation with response_model
    response = {"items": items, "total": total, "page": page, "page_size": page_size}

    # Add warnings if any were generated during ID validation
    if expand and items and "warnings" in locals() and warnings:
        response["warnings"] = warnings

    return response


@router.get("/preflight")
async def preflight_l2a(
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # Add gateway
):
    """
    Quick preflight check to surface overlap and estimated pairs before runs.
    Returns domain overlap, name overlap, and blocking strategy that will be used.
    """
    # Get policy
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()

    # Get connected Salesforce integration
    integ_res = await db.execute(
        select(IntegrationModel).where(
            IntegrationModel.account_id == str(account_id),
            IntegrationModel.provider == IntegrationProvider.SALESFORCE,
            IntegrationModel.status == IntegrationStatus.CONNECTED,
        )
    )
    integ = integ_res.scalar_one_or_none()
    if not integ:
        raise HTTPException(400, "No Salesforce integration connected")
        # No active policy yet � continue to discover options
        return {"error": "No active L2A policy", "lead_count": 0, "acct_count": 0}

    # No longer need to get integration - gateway handles it

    # Gateway handles token persistence automatically

    try:
        # Sample leads and accounts
        lead_fields = ["Id", "Company", "Email", "Website", "Domain__c"]
        account_fields = ["Id", "Name", "Website", "Domain__c"]

        leads = await sf.get_sample_records("Lead", lead_fields, n=1000)
        accounts = await sf.get_sample_records("Account", account_fields, n=5000)

        # Calculate domain overlap
        import pandas as pd
        from ...preprocessing.rules.domain_utils import registrable_domain

        leads_df = pd.DataFrame(leads)
        accounts_df = pd.DataFrame(accounts)

        # Extract domains
        lead_domains = set()
        acct_domains = set()

        if "Email" in leads_df.columns:
            for email in leads_df["Email"].dropna():
                if "@" in str(email):
                    domain = str(email).split("@")[-1].lower()
                    reg_domain = registrable_domain(domain)
                    if reg_domain:
                        lead_domains.add(reg_domain)

        if "Website" in accounts_df.columns:
            for website in accounts_df["Website"].dropna():
                reg_domain = registrable_domain(str(website))
                if reg_domain:
                    acct_domains.add(reg_domain)

        domain_overlap = len(lead_domains & acct_domains) / max(len(lead_domains), 1)

        # Calculate name prefix overlap
        def _norm_prefix(s):
            if not s:
                return ""
            s = str(s).lower()
            for suf in (
                " inc",
                " llc",
                " ltd",
                " corp",
                " corporation",
                " company",
                " co",
            ):
                if s.endswith(suf):
                    s = s[: -len(suf)]
            return s[:8].strip()

        lead_prefixes = set()
        acct_prefixes = set()

        if "Company" in leads_df.columns:
            for company in leads_df["Company"].dropna():
                prefix = _norm_prefix(company)
                if prefix:
                    lead_prefixes.add(prefix)

        if "Name" in accounts_df.columns:
            for name in accounts_df["Name"].dropna():
                prefix = _norm_prefix(name)
                if prefix:
                    acct_prefixes.add(prefix)

        name_overlap_keys = len(lead_prefixes & acct_prefixes)

        # Determine intended fields (simplified)
        intended_fields = [
            "email_domain<->website_domain:Domain",
            "Company<->Name:WRatio",
            "Phone<->Phone:Exact",
        ]

        # Get engine threshold
        engine_threshold = float(os.getenv("L2A_ENGINE_THRESHOLD", "0.75"))

        return {
            "lead_count": len(leads),
            "acct_count": len(accounts),
            "domain_overlap": round(domain_overlap, 3),
            "name_block_overlap_keys": name_overlap_keys,
            "intended_fields": intended_fields,
            "engine_threshold": engine_threshold,
            "blocking_strategy": "domain" if domain_overlap > 0.01 else "name_prefix",
            "warning": "Low overlap; fuzzy recall will be minimal"
            if domain_overlap < 0.01 and name_overlap_keys < 10
            else None,
        }

    except (SalesforceNotConfigured, SalesforceAuthError) as e:
        logger.warning(f"Salesforce not connected for preflight: {e}")
        raise HTTPException(
            status_code=424,
            detail="Salesforce integration is not connected. Please connect Salesforce first.",
        )
    except Exception as e:
        logger.error(f"Preflight error: {e}")
        return {"error": str(e), "lead_count": 0, "acct_count": 0}


# =============================
# Manual Studio (MVP) Endpoints
# =============================


class ManualFilter(BaseModel):
    object: str  # 'Lead' | 'Account'
    field: str
    op: str  # 'contains' | 'equals' | 'not_equals' | 'is_empty' | 'after' | 'before'
    value: Optional[str] = None


class ManualSearchRequest(BaseModel):
    object: str = "Lead"  # 'Lead' | 'Account'
    filters: List[ManualFilter] = []
    limit: int = 50
    page: int = 1


class ManualPreviewRequest(BaseModel):
    lead_id: str
    account_id: str


class ManualPreviewBulkRequest(BaseModel):
    lead_ids: List[str]
    account_id: str


class ManualLinkRequest(BaseModel):
    lead_id: str
    account_id: str
    commit: bool = True
    explanation: Optional[str] = "manual"


# Saved Views DTOs
class SavedViewCreate(BaseModel):
    name: str
    object_type: str  # 'Lead' or 'Account'
    filters: List[Dict[str, Any]]
    scope: str = "user"


class SavedViewUpdate(BaseModel):
    name: Optional[str] = None
    filters: Optional[List[Dict[str, Any]]] = None


# ===== New Manual Studio helpers (cursor-based search) =====
class FilterChip(BaseModel):
    field: str
    op: str
    value: Optional[Any] = None


class SortSpec(BaseModel):
    field: str
    dir: str = "asc"  # 'asc' | 'desc'


class LeadSearchBody(BaseModel):
    filters: List[FilterChip] = []
    page_size: int = 50
    cursor: Optional[str] = None
    sort: Optional[SortSpec] = None


class AccountSearchBody(BaseModel):
    q: Optional[str] = None
    filters: List[FilterChip] = []
    page_size: int = 50
    cursor: Optional[str] = None


def _escape_like(value: str) -> str:
    s = str(value or "")
    # Escape % and _ which are wildcards in LIKE
    s = s.replace("%", "\\%").replace("_", "\\_")
    # Basic single-quote escape
    return s.replace("'", "\\'")


def _escape_literal(value: Any) -> str:
    """Escape single quotes for safe SOQL literal usage."""
    return str(value or "").replace("'", "\\'")


async def _get_field_types(sf: SalesforceGateway, sobject: str) -> dict:
    desc = await sf.describe(sobject)
    return {f.get("name"): f.get("type") for f in desc.get("fields", [])}


def _build_where(filters: List[FilterChip], field_types: dict) -> List[str]:
    parts: List[str] = []
    for f in filters or []:
        name = f.field
        op = (f.op or "").lower()
        val = f.value
        # Skip unknown fields for safety
        if name not in field_types:
            continue
        ftype = (field_types.get(name) or "").lower()

        if op == "equals":
            parts.append(f"{name} = '{_escape_literal(val)}'")
        elif op == "not_equals":
            parts.append(f"{name} != '{_escape_literal(val)}'")
        elif op == "contains" and ftype in {"string", "email", "url", "phone"}:
            parts.append(f"{name} LIKE '%{_escape_like(val)}%'")
        elif op == "starts_with" and ftype in {"string", "email", "url", "phone"}:
            parts.append(f"{name} LIKE '{_escape_like(val)}%'")
        elif op == "is_empty":
            if ftype in {
                "string",
                "email",
                "url",
                "phone",
                "picklist",
                "multipicklist",
            }:
                parts.append(f"({name} = NULL OR {name} = '')")
            else:
                parts.append(f"({name} = NULL)")
        elif op == "in" and isinstance(val, list):
            safe_vals = [f"'{_escape_literal(v)}'" for v in val]
            if safe_vals:
                parts.append(f"{name} IN ({', '.join(safe_vals)})")
        elif op in {"before", "after", "on"} and ftype in {"date", "datetime"}:
            comp = {"before": "<", "after": ">", "on": "="}[op]
            parts.append(f"{name} {comp} '{_escape_literal(val)}'")
        elif op == "last_n_days" and ftype in {"date", "datetime"}:
            try:
                n = int(val)
                parts.append(f"{name} = LAST_N_DAYS:{n}")
            except Exception:
                continue
        else:
            # Fallback equals
            parts.append(f"{name} = '{_escape_literal(val)}'")
    return parts


async def _select_fields_dynamic(
    sf: SalesforceGateway,
    object_type: str,
    policy_writeback: str | None = None,
) -> List[str]:
    desc = await sf.describe(object_type)
    avail = {
        (f.get("name") or "").lower(): (f.get("name") or "")
        for f in desc.get("fields", [])
    }
    if object_type == "Lead":
        base = ["Id", "Company", "Email", "LeadSource", "CampaignId"]
        if policy_writeback:
            base.append(policy_writeback)
        wanted = [avail[x.lower()] for x in base if x.lower() in avail]
        if "id" not in avail:
            raise HTTPException(500, "Lead.Id not available in describe()")
        if not any((w or "").lower() == "company" for w in wanted):
            if "company" in avail:
                wanted.append(avail["company"])
        return wanted or ["Id"]
    else:
        base = [
            "Id",
            "Name",
            "Website",
            "Domain__c",
            "Industry",
            "BillingCountry",
            "BillingCity",
            "BillingState",
            "ParentId",
        ]
        wanted = [avail[x.lower()] for x in base if x.lower() in avail]
        if "id" not in avail or "name" not in avail:
            raise HTTPException(500, "Account Id/Name not available in describe()")
        return wanted or ["Id"]


async def _build_soql(
    sf: SalesforceGateway,
    object_type: str,
    filters: List[FilterChip],
    page_size: int,
    sort: Optional[SortSpec],
    select_fields: List[str] | None = None,
) -> str:
    ftypes = await _get_field_types(sf, object_type)
    where_parts = _build_where(filters, ftypes)
    where = f" WHERE {' AND '.join(where_parts)}" if where_parts else ""
    order = ""
    if sort and sort.field in ftypes:
        direction = "DESC" if (sort.dir or "").lower() == "desc" else "ASC"
        order = f" ORDER BY {sort.field} {direction}"
    limit = max(1, min(int(page_size or 50), 500))
    if not select_fields:
        select_fields = await _select_fields_dynamic(sf, object_type)
    fields = ", ".join(select_fields)
    return f"SELECT {fields} FROM {object_type}{where}{order} LIMIT {limit}"


# ========== Count endpoint for Select-all ==========
class LeadCountBody(BaseModel):
    filters: List[FilterChip] = []


@router.post("/manual/leads/count")
async def manual_leads_count(
    body: LeadCountBody,
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
    account_id: str = Depends(require_account),
):
    ftypes = await _get_field_types(sf, "Lead")
    where = " AND ".join(_build_where(body.filters or [], ftypes))
    soql = "SELECT COUNT() FROM Lead" + (f" WHERE {where}" if where else "")
    res = await sf.soql(soql)
    return {"count": int(res.get("totalSize", 0))}


# ========== Bulk link start + job status ==========
class BulkLinkStart(BaseModel):
    filters: List[FilterChip]
    target_account_id: str
    exclusions: List[str] = []
    dry_run: bool = False


@router.post("/manual/bulk/link/start")
async def bulk_link_start(
    body: BulkLinkStart,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    # Validate target Account exists
    try:
        _ = await sf.get_record("Account", body.target_account_id, fields=["Id"])
    except Exception:
        raise HTTPException(400, "Target account not found or inaccessible")

    # Resolve policy & writeback
    pol = (
        await db.execute(
            select(L2APolicy).where(
                L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
            )
        )
    ).scalar_one_or_none()
    if not pol:
        raise HTTPException(400, "No active L2A policy")
    wb = pol.link_field_api_name or getattr(pol, "lead_lookup_field", None)
    mode = pol.link_overwrite_mode or "if_empty"

    # FLS: writeback field must be updateable for this connection
    if not wb:
        raise HTTPException(400, "No writeback field configured for policy")
    try:
        desc = await sf.describe("Lead")
        fmap = {f.get("name"): f for f in desc.get("fields", [])}
        if wb not in fmap or not fmap[wb].get("updateable", False):
            raise HTTPException(
                400, f"Writeback field {wb} not updateable by this connection"
            )
    except HTTPException:
        raise
    except Exception:
        raise HTTPException(
            424, "Salesforce metadata unavailable for writeback validation"
        )

    # Count first (estimated total)
    cnt = await manual_leads_count(LeadCountBody(filters=body.filters), sf)
    total_est = int(cnt.get("count", 0))
    needs_approval = (total_est >= APPROVAL_THRESHOLD) or (mode == "always")

    # Serialize filters and exclusions safely
    def _ser_filters(fs: List[FilterChip]) -> str:
        try:
            return json.dumps(
                [
                    f.model_dump() if hasattr(f, "model_dump") else dict(f)
                    for f in (fs or [])
                ]
            )
        except Exception:
            return json.dumps([])

    exclusions = list(dict.fromkeys(body.exclusions or []))

    job = L2ABulkJob(
        account_id=str(account_id),
        user_id=str(user_id),
        kind=("manual_link_dryrun" if body.dry_run else "manual_link"),
        target_account_id=body.target_account_id,
        filters_json=_ser_filters(body.filters or []),
        exclusions_json=json.dumps(exclusions),
        writeback_field=wb,
        overwrite_mode=mode,
        total=total_est,
        status=("queued" if not needs_approval else "waiting_approval"),
        approval_status=("auto" if not needs_approval else "waiting_approval"),
        requested_by=str(user_id),
    )
    db.add(job)
    await db.commit()

    # Enqueue only if no approval needed
    if not needs_approval:
        await _enqueue_if_capacity(job.id, db)
    else:
        try:
            org_name = str(account_id)
            requester = str(user_id)
            await notify_approval_request(org_name, requester, total_est, job.id)
        except Exception:
            pass

    return {
        "job_id": job.id,
        "status": job.status,
        "approval_status": job.approval_status,
        "total": job.total,
    }


@router.get("/jobs/{job_id}/status")
async def job_status(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    row = (
        await db.execute(
            select(L2ABulkJob).where(
                L2ABulkJob.id == job_id,
                L2ABulkJob.account_id == str(account_id),
                L2ABulkJob.user_id == str(user_id),
            )
        )
    ).scalar_one_or_none()
    if not row:
        raise HTTPException(404, "Job not found")
    pct = (
        0 if row.total == 0 else min(100, int(100 * row.processed / max(1, row.total)))
    )
    return {
        "job_id": row.id,
        "status": row.status,
        "approval_status": getattr(row, "approval_status", None),
        "progress_pct": pct,
        "total": row.total,
        "processed": row.processed,
        "applied": row.applied,
        "skipped": row.skipped,
        "errors": row.error_count,
        "eta_sec": getattr(row, "eta_sec", 0),
        "blocked_if_empty": getattr(row, "blocked_if_empty", 0),
        "blocked_never": getattr(row, "blocked_never", 0),
        "writeback_field": row.writeback_field,
        "overwrite_mode": row.overwrite_mode,
        "created_at": row.created_at.isoformat() if row.created_at else None,
        "updated_at": row.updated_at.isoformat() if row.updated_at else None,
        "started_at": row.started_at.isoformat()
        if getattr(row, "started_at", None)
        else None,
        "ended_at": row.ended_at.isoformat()
        if getattr(row, "ended_at", None)
        else None,
    }


@router.get("/jobs/{job_id}/events")
async def job_events(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    async def gen():
        last = None
        while True:
            row = (
                await db.execute(
                    select(L2ABulkJob).where(
                        L2ABulkJob.id == job_id,
                        L2ABulkJob.account_id == str(account_id),
                        L2ABulkJob.user_id == str(user_id),
                    )
                )
            ).scalar_one_or_none()
            if not row:
                yield f"event: done\ndata: {json.dumps({'error':'not_found'})}\n\n"
                break
            cur = {
                "status": row.status,
                "approval_status": getattr(row, "approval_status", None),
                "total": row.total,
                "processed": row.processed,
                "applied": row.applied,
                "skipped": row.skipped,
                "errors": row.error_count,
                "eta_sec": getattr(row, "eta_sec", 0),
                "blocked_if_empty": getattr(row, "blocked_if_empty", 0),
                "blocked_never": getattr(row, "blocked_never", 0),
                "progress_pct": 0
                if row.total == 0
                else min(100, int(100 * row.processed / max(1, row.total))),
                "updated_at": row.updated_at.isoformat() if row.updated_at else None,
                "started_at": row.started_at.isoformat()
                if getattr(row, "started_at", None)
                else None,
                "ended_at": row.ended_at.isoformat()
                if getattr(row, "ended_at", None)
                else None,
            }
            if cur != last:
                yield f"data: {json.dumps(cur)}\n\n"
                last = cur
            if row.status in ("completed", "failed", "canceled"):
                yield "event: done\ndata: {}\n\n"
                break
            await asyncio.sleep(1.0)

    return StreamingResponse(gen(), media_type="text/event-stream")


@router.post("/jobs/{job_id}/cancel", dependencies=[Depends(require_admin_or_manager)])
async def job_cancel(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    row = (
        await db.execute(
            select(L2ABulkJob).where(
                L2ABulkJob.id == job_id,
                L2ABulkJob.account_id == str(account_id),
                L2ABulkJob.user_id == str(user_id),
            )
        )
    ).scalar_one_or_none()
    if not row:
        raise HTTPException(404, "Job not found")
    if row.status not in ("queued", "running"):
        return {"ok": True, "status": row.status}
    row.status = "canceled"
    await db.commit()
    return {"ok": True}


@router.post("/jobs/{job_id}/retry", dependencies=[Depends(require_admin_or_manager)])
async def job_retry(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    row = (
        await db.execute(
            select(L2ABulkJob).where(
                L2ABulkJob.id == job_id,
                L2ABulkJob.account_id == str(account_id),
                L2ABulkJob.user_id == str(user_id),
            )
        )
    ).scalar_one_or_none()
    if not row:
        raise HTTPException(404, "Job not found")
    if row.status not in ("failed", "canceled", "completed"):
        return {"ok": False, "reason": f"Cannot retry from {row.status}"}
    row.status = "queued"
    row.processed = row.applied = row.skipped = row.error_count = 0
    row.last_cursor = None
    row.last_page_size = 0
    row.eta_sec = 0
    row.blocked_if_empty = 0
    row.blocked_never = 0
    row.started_at = None
    row.ended_at = None
    await db.commit()
    await _enqueue_if_capacity(row.id, db)
    return {"ok": True, "status": row.status}


@router.get("/jobs/{job_id}/report.csv")
async def job_error_report(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    row = (
        await db.execute(
            select(L2ABulkJob).where(
                L2ABulkJob.id == job_id,
                L2ABulkJob.account_id == str(account_id),
                L2ABulkJob.user_id == str(user_id),
            )
        )
    ).scalar_one_or_none()
    if not row:
        raise HTTPException(404, "Job not found")
    if not row.error_report:
        return PlainTextResponse(
            "lead_id,company,email,previous_value,error\n", media_type="text/csv"
        )
    try:
        items = json.loads(row.error_report)
    except Exception:
        return PlainTextResponse(
            "lead_id,company,email,previous_value,error\n", media_type="text/csv"
        )
    lines = ["lead_id,company,email,previous_value,error"]
    for it in items:
        lid = str(it.get("lead_id", "") or "")
        company = str(it.get("company", "") or "").replace(",", ";")
        email = str(it.get("email", "") or "").replace(",", ";")
        prev = str(it.get("prev", "") or "").replace(",", ";")
        err = str(it.get("error", "") or "").replace(",", ";")
        lines.append(f"{lid},{company},{email},{prev},{err}")
    return PlainTextResponse("\n".join(lines) + "\n", media_type="text/csv")


# ========== Secure Job Center list (admin/manager only) ==========


@router.get("/jobs", dependencies=[Depends(require_admin_or_manager)])
async def list_jobs(
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    limit: int = 50,
):
    # Enforce role: admin or manager (steward). Accept both Enum and string representations.
    me = (
        await db.execute(
            select(User).where(
                User.id == str(user_id), User.account_id == str(account_id)
            )
        )
    ).scalar_one_or_none()
    role_ok = False
    if me and getattr(me, "role", None) is not None:
        r = me.role
        resolved_role = UserRole.from_raw(r)
        if resolved_role:
            role_ok = resolved_role in (
                {UserRole.STEWARD} | set(UserRole.admin_roles())
            )
        else:
            rv = str(r).strip().lower()
            role_ok = rv in ({"manager", "steward"} | UserRole.admin_values())
    if not role_ok:
        # In development, allow access to simplify local testing
        if os.getenv("ENV", "").lower() in ("dev", "development"):
            role_ok = True
        else:
            raise HTTPException(403, "Insufficient role")

    q = (
        select(L2ABulkJob)
        .where(L2ABulkJob.account_id == str(account_id))
        .order_by(L2ABulkJob.created_at.desc())
        .limit(min(int(limit or 50), 200))
    )
    rows = (await db.execute(q)).scalars().all()
    out = []
    for r in rows:
        out.append(
            {
                "id": r.id,
                "status": r.status,
                "approval_status": getattr(r, "approval_status", None),
                "total": r.total,
                "processed": r.processed,
                "applied": r.applied,
                "skipped": r.skipped,
                "errors": r.error_count,
                "created_at": r.created_at.isoformat() if r.created_at else None,
                "updated_at": r.updated_at.isoformat() if r.updated_at else None,
            }
        )
    return out


# ========== Approvals ==========
class ApproveBody(BaseModel):
    approve: bool


@router.post("/jobs/{job_id}/approve", dependencies=[Depends(require_admin_or_manager)])
async def job_approve(
    job_id: str,
    body: ApproveBody,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    row = await db.get(L2ABulkJob, job_id)
    if not row or row.account_id != str(account_id):
        raise HTTPException(404, "Job not found")
    if row.status != "waiting_approval":
        return {"ok": False, "reason": f"Not waiting approval (status={row.status})"}
    row.approval_status = "approved" if body.approve else "rejected"
    row.approved_by = str(user_id)
    if body.approve:
        row.status = "queued"
        await db.commit()
        await _enqueue_if_capacity(job_id, db)
    else:
        row.status = "canceled"
        await db.commit()
    return {"ok": True, "status": row.status}


# ========== Bulk Undo by Job ==========
@router.post("/jobs/{job_id}/undo", dependencies=[Depends(require_admin_or_manager)])
async def job_undo(
    job_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    from ...model_defs.l2a_models import L2ASuggestion

    # Enforce undo TTL via settings
    from ... import settings

    cutoff = utcnow() - _dt.timedelta(hours=settings.L2A_UNDO_TTL_HOURS)
    sug_q = select(L2ASuggestion.id).where(
        L2ASuggestion.job_id == job_id,
        L2ASuggestion.status == "linked",
        L2ASuggestion.decided_at >= cutoff,
    )
    ids = [str(r[0]) for r in (await db.execute(sug_q)).all()]
    if not ids:
        return {"undone": 0}
    undone, errs = await undo_suggestions_batch(ids, db, sf)
    return {"undone": undone, "errors": len(errs or [])}


async def undo_suggestions_batch(
    ids: List[str], db: AsyncSession, sf: SalesforceGateway
):
    from ...model_defs.l2a_models import L2ASuggestion

    undone = 0
    errors = []
    for sid in ids or []:
        try:
            s = await db.get(L2ASuggestion, sid)
            if not s or s.status != "linked":
                continue
            from ...services.sf_fls import assert_updateable

            await assert_updateable(sf, "Lead", s.applied_field)
            await sf.update("Lead", s.lead_id, {s.applied_field: s.previous_value})
            s.status = "pending"
            s.decided_at = None
            s.decided_by = None
            s.applied_value = None
            await db.flush()
            undone += 1
        except Exception as e:
            errors.append({"id": sid, "error": str(e)})
    await db.commit()
    return undone, errors


# ========== Helper: enqueue if capacity ==========
async def _enqueue_if_capacity(job_id: str, db: AsyncSession):
    j = await db.get(L2ABulkJob, job_id)
    if not j:
        return {"ok": False}
    running = (
        await db.execute(
            select(func.count())
            .select_from(L2ABulkJob)
            .where(
                L2ABulkJob.account_id == j.account_id, L2ABulkJob.status == "running"
            )
        )
    ).scalar_one()
    if running >= CONCURRENCY_LIMIT:
        return {"queued": True}
    updated = (
        await db.execute(
            update(L2ABulkJob)
            .where(L2ABulkJob.id == job_id, L2ABulkJob.status.in_(["queued"]))
            .values(status="running", started_at=utcnow())
            .returning(L2ABulkJob.id)
        )
    ).scalar_one_or_none()
    await db.commit()
    if updated:
        asyncio.create_task(run_bulk_link_job(job_id))
    return {"started": bool(updated)}


# ========== Explore + Feedback ==========
class ExploreBody(BaseModel):
    min_score: float = 0.5
    max_score: float = 0.7
    limit: int = 50
    filters: List[FilterChip] = []


@router.post("/explore/sample")
async def explore_sample(
    body: ExploreBody,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
):
    q = (
        select(L2ASuggestion)
        .where(
            L2ASuggestion.policy_id.in_(
                select(L2APolicy.id).where(L2APolicy.account_id == str(account_id))
            ),
            L2ASuggestion.status == "pending",
            L2ASuggestion.score >= float(body.min_score or 0.0),
            L2ASuggestion.score <= float(body.max_score or 1.0),
        )
        .order_by(func.random())
        .limit(min(int(body.limit or 50), 200))
    )
    rows = (await db.execute(q)).scalars().all()
    return [
        {
            "id": str(r.id),
            "lead_id": r.lead_id,
            "account_id": r.account_id,
            "score": r.score,
            # Reuse reasons as explanations for now
            "explanations": r.reasons or {},
            "reasons": r.reasons or {},
        }
        for r in rows
    ]


class FeedbackBody(BaseModel):
    suggestion_id: str
    verdict: str  # 'correct'|'incorrect'
    reason: Optional[str] = None


@router.post("/feedback")
async def l2a_feedback(
    body: FeedbackBody,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    # Map to DecisionLogger outcomes
    outcome = {
        "correct": "accepted",
        "incorrect": "reverted",
    }.get((body.verdict or "").lower(), "accepted")

    try:
        logger = DecisionLogger()
        await logger.log_feedback(
            job_id="explore-sample",
            match_id=body.suggestion_id,
            outcome=outcome,
            user_id=str(user_id),
            corrected_target=None,
        )
    except Exception:
        # Non-critical
        pass

    # Optionally update suggestion (skip if model lacks columns)
    try:
        s = await db.get(L2ASuggestion, body.suggestion_id)
        if s:
            # Attach lightweight markers to reasons dict
            rs = s.reasons or {}
            if isinstance(rs, str):
                try:
                    rs = json.loads(rs)
                except Exception:
                    rs = {}
            rs["_feedback"] = body.verdict
            if body.reason:
                rs["_feedback_reason"] = body.reason
            s.reasons = rs
            await db.commit()
    except Exception:
        pass

    return {"ok": True}


@router.post("/manual/leads/search")
async def manual_leads_search(
    body: LeadSearchBody,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    # Cursor fetch
    if body.cursor:
        try:
            page = await sf.fetch_next(body.cursor)
            items = [r for r in page.get("records", [])]
            return {
                "items": items,
                "next_cursor": page.get("nextRecordsUrl"),
            }
        except Exception as e:
            raise HTTPException(500, f"Cursor fetch failed: {e}")

    # Initial query
    # Include org-configured writeback field in SELECT to power "Linked?" column
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    wb = (
        (policy.link_field_api_name or getattr(policy, "lead_lookup_field", None))
        if policy
        else None
    )
    select_fields = await _select_fields_dynamic(sf, "Lead", wb)
    soql = await _build_soql(
        sf, "Lead", body.filters or [], body.page_size or 50, body.sort, select_fields
    )
    try:
        res = await sf.soql(soql)
        return {
            "items": res.get("records", []),
            "next_cursor": res.get("nextRecordsUrl"),
        }
    except Exception as e:
        logger.error(f"Lead search failed: {e}")
        raise HTTPException(500, "Lead search failed")


@router.post("/manual/accounts/search")
async def manual_accounts_search(
    body: AccountSearchBody,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    # Cursor fetch
    if body.cursor:
        try:
            page = await sf.fetch_next(body.cursor)
            items = [r for r in page.get("records", [])]
            return {
                "items": items,
                "next_cursor": page.get("nextRecordsUrl"),
            }
        except Exception as e:
            raise HTTPException(500, f"Cursor fetch failed: {e}")

    filters = body.filters or []
    # Add simple q OR condition across Name/Website and Domain__c if available
    if body.q:
        qtext = _escape_like(body.q)
        desc = await sf.describe("Account")
        avail = {
            (f.get("name") or "").lower(): (f.get("name") or "")
            for f in desc.get("fields", [])
        }
        ors = []
        # Always attempt Name/Website when present
        for f in ["Name", "Website"]:
            if f.lower() in avail:
                ors.append(f"{avail[f.lower()]} LIKE '%{qtext}%'")
        # Include Domain__c only if present in org
        if "domain__c" in avail:
            ors.append(f"{avail['domain__c']} LIKE '%{qtext}%'")
        ftypes = await _get_field_types(sf, "Account")
        where_parts = _build_where(filters, ftypes)
        if ors:
            where_parts.append(f"({ ' OR '.join(ors) })")
        where = f" WHERE {' AND '.join(where_parts)}" if where_parts else ""
        select_fields = await _select_fields_dynamic(sf, "Account")
        soql = f"SELECT {', '.join(select_fields)} FROM Account{where} LIMIT {max(1, min(int(body.page_size or 50), 500))}"
    else:
        select_fields = await _select_fields_dynamic(sf, "Account")
        soql = await _build_soql(
            sf, "Account", filters, body.page_size or 50, None, select_fields
        )

    try:
        res = await sf.soql(soql)
        return {
            "items": res.get("records", []),
            "next_cursor": res.get("nextRecordsUrl"),
        }
    except Exception as e:
        logger.error(f"Account search failed: {e}")
        raise HTTPException(500, "Account search failed")


def _soql_quote(v: str) -> str:
    if v is None:
        return "null"
    # Basic SOQL escaping for single quotes
    return "'" + str(v).replace("'", "\\'") + "'"


def _to_soql_condition(f: ManualFilter) -> str:
    fld = f.field
    op = f.op.lower()
    val = f.value
    if op == "contains":
        return f"{fld} LIKE '%{str(val or '').replace('%', '')}%'"
    if op == "equals":
        return f"{fld} = {_soql_quote(val)}"
    if op == "not_equals":
        return f"{fld} != {_soql_quote(val)}"
    if op == "is_empty":
        return f"({fld} = null)"
    if op == "after":
        return f"{fld} > {_soql_quote(val)}"
    if op == "before":
        return f"{fld} < {_soql_quote(val)}"
    # Fallback - treat as equals
    return f"{fld} = {_soql_quote(val)}"


@router.post("/manual/search")
async def manual_search(
    payload: ManualSearchRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Search Salesforce records using simple filter chips.

    Maps client filters into a safe SOQL WHERE clause and returns basic fields.
    """
    sobj = (payload.object or "Lead").strip()
    if sobj not in ("Lead", "Account"):
        raise HTTPException(400, "object must be 'Lead' or 'Account'")

    # Fields to project by object
    if sobj == "Lead":
        fields = [
            "Id",
            "Company",
            "Email",
            "Phone",
            "City",
            "State",
            "Account__c",
            # Removed LastModifiedDate - gateway blocks it
        ]
    else:
        fields = [
            "Id",
            "Name",
            "Website",
            "Phone",
            "BillingCity",
            "BillingState",
            # Removed LastModifiedDate - gateway blocks it
        ]

    where_parts = []
    for flt in payload.filters or []:
        try:
            where_parts.append(_to_soql_condition(flt))
        except Exception:
            continue
    where = f" WHERE {' AND '.join(where_parts)}" if where_parts else ""

    order = ""  # Removed ORDER BY LastModifiedDate - gateway blocks it
    limit = max(1, min(int(payload.limit or 50), 200))
    soql = f"SELECT {', '.join(fields)} FROM {sobj}{where}{order} LIMIT {limit}"

    try:
        res = await sf.soql(soql)
        recs = res.get("records", []) if isinstance(res, dict) else (res or [])
        return {
            "total": res.get("totalSize", len(recs))
            if isinstance(res, dict)
            else len(recs),
            "items": recs,
            "soql": soql,
        }
    except SalesforceConnectionError as e:
        raise HTTPException(424, f"Salesforce not connected: {e}")
    except Exception as e:
        logger.error(f"manual_search error: {e}")
        raise HTTPException(500, f"Search failed: {e}")


@router.post("/manual/preview")
async def manual_preview(
    body: dict,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Return pre-flight diffs for a manual link (single or bulk)."""
    # Normalize body into either single or bulk request
    diffs = []
    try:
        if "lead_ids" in body and isinstance(body.get("lead_ids"), list):
            bulk = ManualPreviewBulkRequest(**body)
            lead_ids = bulk.lead_ids
            account = bulk.account_id
        else:
            single = ManualPreviewRequest(**body)
            lead_ids = [single.lead_id]
            account = single.account_id
    except Exception as e:
        raise HTTPException(400, f"Invalid preview payload: {e}")
    # Load policy
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    if not policy:
        raise HTTPException(400, "No active L2A policy for this account")

    writeback_field = policy.link_field_api_name or policy.lead_lookup_field
    if not writeback_field:
        raise HTTPException(400, "No writeback field configured")

    overwrite_mode = policy.link_overwrite_mode or "if_empty"

    # Fetch current values and compute per-lead diffs
    for lid in lead_ids:
        try:
            lead = await sf.get_record("Lead", lid, fields=[writeback_field])
            prev = lead.get(writeback_field)
        except Exception:
            prev = None

        reason_code = "assign"
        if prev and prev != account:
            reason_code = "upgrade"

        blocked_reason = None
        if overwrite_mode == "never" and prev:
            blocked_reason = "Field already populated (mode=never)"
        elif overwrite_mode == "if_empty" and prev and prev != account:
            blocked_reason = f"Field already set to {prev} (mode=if_empty)"

        diffs.append(
            {
                "lead_id": lid,
                "field": writeback_field,
                "before": prev,
                "after": account,
                **({"blocked_reason": blocked_reason} if blocked_reason else {}),
            }
        )

    return {
        "diffs": diffs,
        "writeback_field": writeback_field,
        "overwrite_mode": overwrite_mode,
    }


@router.post("/manual/suggestion")
async def create_manual_suggestion(
    body: ManualLinkRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Create a manual L2A suggestion that can be applied via /apply."""
    # Load policy
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    if not policy:
        raise HTTPException(400, "No active L2A policy for this account")

    writeback_field = policy.link_field_api_name or policy.lead_lookup_field
    if not writeback_field:
        raise HTTPException(400, "No writeback field configured")

    # Determine incumbent to set reason_code
    try:
        lead = await sf.get_record("Lead", body.lead_id, fields=[writeback_field])
        incumbent = lead.get(writeback_field)
    except Exception:
        incumbent = None

    reason_code = "assign"
    incumbent_account_id = None
    if incumbent and incumbent != body.account_id:
        reason_code = "upgrade"
        incumbent_account_id = incumbent

    s = L2ASuggestion(
        policy_id=policy.id,
        lead_id=body.lead_id,
        account_id=body.account_id,
        explanation=(body.explanation or "manual"),
        status="pending",
        reason_code=reason_code,
        incumbent_account_id=incumbent_account_id,
    )
    db.add(s)
    await db.flush()
    await db.commit()
    return {"suggestion_id": s.id}


class ManualSuggestionsRequest(BaseModel):
    lead_ids: List[str]
    account_id: str


@router.post("/manual/suggestions")
async def create_manual_suggestions(
    body: ManualSuggestionsRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Bulk-create manual L2A suggestions and return their IDs.

    Skips records blocked by overwrite policy with a clear reason.
    """
    # Validate account exists (avoid 404 later)
    try:
        _ = await sf.get_record("Account", body.account_id, fields=["Id"])
    except Exception:
        raise HTTPException(400, "Account not found or inaccessible")

    # Load policy
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    if not policy:
        raise HTTPException(400, "No active L2A policy for this account")

    writeback_field = policy.link_field_api_name or policy.lead_lookup_field
    if not writeback_field:
        raise HTTPException(400, "No writeback field configured")

    overwrite_mode = policy.link_overwrite_mode or "if_empty"

    created_ids: List[str] = []
    skipped: List[dict] = []

    # Fetch incumbents in a simple loop (batching not essential here)
    for lid in list(dict.fromkeys(body.lead_ids or [])):
        try:
            lead = await sf.get_record("Lead", lid, fields=[writeback_field])
            prev = lead.get(writeback_field)
        except Exception:
            prev = None

        # Policy-driven skip logic
        block_reason = None
        if overwrite_mode == "never" and prev:
            block_reason = "Field already populated (mode=never)"
        elif overwrite_mode == "if_empty" and prev and prev != body.account_id:
            block_reason = f"Field already set to {prev} (mode=if_empty)"

        if block_reason:
            skipped.append({"lead_id": lid, "reason": block_reason})
            continue

        reason_code = "assign"
        incumbent_account_id = None
        if prev and prev != body.account_id:
            reason_code = "upgrade"
            incumbent_account_id = prev

        s = L2ASuggestion(
            policy_id=policy.id,
            lead_id=lid,
            account_id=body.account_id,
            explanation="manual",
            status="pending",
            reason_code=reason_code,
            incumbent_account_id=incumbent_account_id,
        )
        db.add(s)
        await db.flush()
        created_ids.append(s.id)

    await db.commit()

    return {
        "suggestion_ids": created_ids,
        "created": len(created_ids),
        "skipped": skipped,
        "writeback_field": writeback_field,
    }


@router.post("/manual/link")
async def manual_link(
    body: ManualLinkRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Thin wrapper: create manual suggestion then optionally apply it."""
    # Create suggestion
    res = await create_manual_suggestion(
        body=body, db=db, account_id=account_id, user_id=user_id, sf=sf
    )
    sugg_id = res.get("suggestion_id")
    out = {"suggestion_id": sugg_id, "linked": False}
    if body.commit:
        apply_res = await apply_suggestions(
            suggestion_ids=[sugg_id],
            db=db,
            account_id=account_id,
            user_id=user_id,
            sf=sf,
        )
        out.update({"linked": True, "apply_result": apply_res})
        await db.commit()
    return out


@router.post("/apply")
async def apply_suggestions(
    suggestion_ids: List[str],
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # Add gateway
):
    # Handle empty arrays gracefully
    if not suggestion_ids:
        return {"updated": 0, "message": "No suggestions provided"}

    # load policy & sf
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    # No active policy yet � continue to discover options

    # Get connected Salesforce integration
    integ_res = await db.execute(
        select(IntegrationModel).where(
            IntegrationModel.account_id == str(account_id),
            IntegrationModel.provider == IntegrationProvider.SALESFORCE,
            IntegrationModel.status == IntegrationStatus.CONNECTED,
        )
    )
    integ = integ_res.scalar_one_or_none()
    if not integ:
        raise HTTPException(400, "No Salesforce integration connected")
    # No active policy yet � continue to discover options

    # Use new writeback field or fall back to old field
    writeback_field = policy.link_field_api_name or policy.lead_lookup_field
    if not writeback_field:
        raise HTTPException(400, "No Lead->Account lookup field configured")

    overwrite_mode = policy.link_overwrite_mode or "if_empty"

    # SF gateway is injected via dependency - no need to create client

    # Gateway handles token persistence automatically

    # Create EventEmitter and ChangeSet for this batch
    emitter = EventEmitter()
    cs_id = f"cs_{uuid.uuid4().hex[:12]}"

    # Create ChangeSet with all required fields (matching canonical definition)
    cs = ORMChangeSet(
        id=cs_id,
        account_id=str(account_id),
        name="L2A Apply",
        type="l2a_apply",
        source="ui",
        actor_type="user",
        actor_id=str(user_id) if user_id else None,
        scope={},  # optional: could add campaign_id, filters, etc.
        policy_id=getattr(policy, "id", None),
        status="open",
        attempted_count=0,
        applied_count=0,
        failed_count=0,
        # created_at will use default=datetime.utcnow from model
    )

    # Add to session with error handling
    try:
        db.add(cs)
        await db.flush()  # Ensure it's persisted to get the ID
    except Exception as e:
        # makes the failure obvious in logs & to the caller
        raise HTTPException(500, f"ChangeSet add failed (mapping/registry?): {e}")

    # fetch suggestions
    q = select(L2ASuggestion).where(
        L2ASuggestion.id.in_(suggestion_ids), L2ASuggestion.status == "pending"
    )
    res = await db.execute(q)
    applied, errors = 0, []
    now = datetime.utcnow()
    events = []

    for s in res.scalars().all():
        try:
            # read old value (for undo)
            lead = await sf.get_record("Lead", s.lead_id, fields=[writeback_field])
            prev = lead.get(writeback_field)

            # Apply overwrite mode logic
            should_write = False
            skip_reason = None

            if overwrite_mode == "always":
                should_write = True
            elif overwrite_mode == "never":
                if not prev:
                    should_write = True
                else:
                    skip_reason = "Field already populated (mode=never)"
            else:  # if_empty (default)
                if not prev or prev == s.account_id:
                    should_write = True
                else:
                    skip_reason = f"Field already set to {prev} (mode=if_empty)"

            if not should_write:
                s.status = "error"
                s.error_message = skip_reason
                errors.append({"lead_id": s.lead_id, "error": s.error_message})
                continue

            # FIX 7: Re-read current value to check for concurrent changes
            lead = await sf.get_record("Lead", s.lead_id, fields=[writeback_field])
            current_value = lead.get(writeback_field)

            # For upgrades, check if incumbent has changed
            if s.reason_code == "upgrade" and s.incumbent_account_id:
                if current_value != s.incumbent_account_id:
                    s.status = "error"
                    s.error_message = f"Stale suggestion: Lead's account changed from {s.incumbent_account_id} to {current_value}"
                    errors.append(
                        {
                            "lead_id": s.lead_id,
                            "error": "Stale suggestion - account was modified by another process",
                        }
                    )
                    continue

            # write lookup
            from ...services.sf_fls import assert_updateable

            await assert_updateable(sf, "Lead", writeback_field)
            await sf.update("Lead", s.lead_id, {writeback_field: s.account_id})
            # mark
            s.status = "linked"
            s.decided_at = now
            s.decided_by = user_id
            s.applied_field = writeback_field
            s.previous_value = prev
            s.applied_value = s.account_id
            applied += 1

            # Emit event for this change
            events.append(
                {
                    "changeset_id": cs_id,
                    "object_type": "Lead",
                    "object_id": s.lead_id,
                    "op": "update_field",
                    "field": writeback_field,
                    "before_value": prev,
                    "after_value": s.account_id,
                    "confidence": float(s.score or 0.0),
                    "tier": s.tier,
                    "suggestion_id": s.id,
                    "reason_text": s.explanation,
                    "field_score_details": s.field_score_details or [],
                    "actor_type": "user",
                    "actor_id": user_id,
                }
            )
        except Exception as e:
            s.status = "error"
            s.error_message = str(e)
            errors.append({"id": s.id, "error": str(e)})

    # Emit batch of events and finalize changeset
    if events:
        await emitter.emit_batch(db, str(account_id), events)

    cs.status = "finalized"
    cs.attempted_count = len(events) + len(errors)
    cs.applied_count = applied
    cs.failed_count = len(errors)
    cs.finalized_at = _dt.datetime.utcnow()

    await db.commit()
    return {"applied": applied, "errors": errors}


@router.post("/reject")
async def reject_suggestions(
    suggestion_ids: List[str],
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # Add gateway
):
    # Handle empty arrays gracefully
    if not suggestion_ids:
        return {"rejected": 0, "message": "No suggestions provided"}

    q = select(L2ASuggestion).where(L2ASuggestion.id.in_(suggestion_ids))
    res = await db.execute(q)
    now = datetime.utcnow()
    n = 0
    for s in res.scalars().all():
        s.status = "rejected"
        s.decided_at = now
        s.decided_by = user_id
        n += 1
    await db.commit()
    return {"rejected": n}


# Writeback field configuration endpoints
class WritebackConfigRequest(BaseModel):
    field_api_name: str
    overwrite_mode: str = "if_empty"

    @field_validator("overwrite_mode")
    @classmethod
    def validate_overwrite_mode(cls, v):
        if v not in ["if_empty", "always", "never"]:
            raise ValueError("overwrite_mode must be one of: if_empty, always, never")
        return v


# ========== Safe Apply (approve-safe) ==========
@router.post("/safe-apply", dependencies=[Depends(require_admin_or_manager)])
async def l2a_safe_apply(
    min_score: float = 0.80,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Apply high-confidence L2A suggestions in bulk.

    Uses existing apply_suggestions path; enforces a score threshold.
    Overwrite behavior is governed by the active L2A policy (default if_empty).
    """
    from ...model_defs.l2a_models import L2ASuggestion

    ids = [
        str(r[0])
        for r in (
            await db.execute(
                select(L2ASuggestion.id).where(
                    L2ASuggestion.status == "pending",
                    (L2ASuggestion.score >= float(min_score)),
                    L2ASuggestion.account_id == str(account_id),
                )
            )
        ).all()
    ][:2000]
    applied = 0
    errors = 0
    for i in range(0, len(ids), 100):
        chunk = ids[i : i + 100]
        if not chunk:
            continue
        res = await apply_suggestions(
            suggestion_ids=chunk, db=db, account_id=account_id, user_id=user_id, sf=sf
        )
        try:
            applied += int(res.get("applied", 0))
            errors += int(len(res.get("errors", []) or []))
        except Exception:
            pass
    return {"applied": applied, "errors": errors}


@router.get("/writeback/options")
async def get_writeback_options(
    db: AsyncSession = Depends(get_session), account_id: str = Depends(require_account)
):
    """List available Salesforce fields for L2A writeback"""
    logger.info(f"get_writeback_options() for account {account_id}")

    # Find ACTIVE policy for this tenant
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    # No active policy yet � continue to discover options

    # Get connected Salesforce integration
    integ_res = await db.execute(
        select(IntegrationModel).where(
            IntegrationModel.account_id == str(account_id),
            IntegrationModel.provider == IntegrationProvider.SALESFORCE,
            IntegrationModel.status == IntegrationStatus.CONNECTED,
        )
    )
    integ = integ_res.scalar_one_or_none()
    if not integ:
        # Safe defaults if no integration connected yet
        return {"options": [], "current_field": None, "current_mode": "if_empty"}

        # No active policy yet � continue to discover options
        # No active policy yet: still discover candidate fields so user can pick one
        service = IntelligentL2AService(db, account_id)
        try:
            res = await service.list_writeback_options_for_integration(integ, None)
            options = res.get("candidates", [])
            return {
                "options": [
                    c if isinstance(c, dict) else c.model_dump() for c in options
                ],
                "current_field": None,
                "current_mode": "if_empty",
            }
        except Exception:
            return {"options": [], "current_field": None, "current_mode": "if_empty"}

    # Get writeback field options from service
    service = IntelligentL2AService(db, account_id)
    try:
        res = await service.list_writeback_options_for_integration(integ, policy)
        options = res.get("candidates", [])

        return {
            "options": [
                c if isinstance(c, dict) else c.model_dump() for c in options
            ],  # [{ apiName, label, updateable, accessible }]
            "current_field": (policy.link_field_api_name if policy else None),
            "current_mode": (policy.link_overwrite_mode if policy else None)
            or "if_empty",
        }
    except Exception as e:
        logger.error(f"Failed to get writeback options: {e}")
        # Return safe defaults on error
        return {
            "options": [],
            "current_field": (policy.link_field_api_name if policy else None),
            "current_mode": (policy.link_overwrite_mode if policy else None)
            or "if_empty",
        }


@router.post("/writeback/choose")
async def choose_writeback_field(
    config: WritebackConfigRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
):
    """Choose and configure the Lead writeback field for L2A"""
    # Get active policy
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    if not policy:
        raise HTTPException(400, "No active L2A policy")

    # Update policy with chosen field
    service = IntelligentL2AService(db, account_id)
    result = await service.choose_writeback_field(
        str(policy.id),
        config.field_api_name,
        OverwriteMode(config.overwrite_mode),
    )

    # Update policy in database
    policy.link_field_api_name = config.field_api_name
    policy.link_overwrite_mode = config.overwrite_mode
    await db.commit()

    return {
        "success": True,
        "field": config.field_api_name,
        "mode": config.overwrite_mode,
        **result,
    }


@router.get("/writeback/validate")
async def validate_writeback(
    sample_lead_id: Optional[str] = None,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
):
    """Validate current writeback configuration"""
    # Get active policy
    pol_res = await db.execute(
        select(L2APolicy).where(
            L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
        )
    )
    policy = pol_res.scalar_one_or_none()
    if not policy:
        raise HTTPException(400, "No active L2A policy")

    # Get connected Salesforce integration
    integ_res = await db.execute(
        select(IntegrationModel).where(
            IntegrationModel.account_id == str(account_id),
            IntegrationModel.provider == IntegrationProvider.SALESFORCE,
            IntegrationModel.status == IntegrationStatus.CONNECTED,
        )
    )
    integ = integ_res.scalar_one_or_none()
    if not integ:
        raise HTTPException(400, "No Salesforce integration connected")

    # Validate writeback
    service = IntelligentL2AService(db, account_id)
    result = await service.validate_writeback(
        integ,
        policy.link_field_api_name,
        sample_lead_id or "",  # service will gracefully handle fetch errors
    )

    return result


@router.post("/undo/{suggestion_id}")
async def undo_link(
    suggestion_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # Add gateway
):
    # revert the lookup value to previous_value
    # SF gateway is injected via dependency - no need to create client

    # Gateway handles token persistence automatically

    res = await db.execute(
        select(L2ASuggestion).where(L2ASuggestion.id == suggestion_id)
    )
    s = res.scalar_one_or_none()
    if not s or s.status != "linked":
        raise HTTPException(400, "Suggestion not linked or not found")
    # Enforce undo TTL for single-undo
    from ... import settings as _settings

    cutoff = utcnow() - timedelta(hours=_settings.L2A_UNDO_TTL_HOURS)
    if not s.decided_at or s.decided_at < cutoff:
        raise HTTPException(400, "Undo window has expired")
    from ...services.sf_fls import assert_updateable

    await assert_updateable(sf, "Lead", s.applied_field)
    await sf.update("Lead", s.lead_id, {s.applied_field: s.previous_value})
    s.status = "pending"
    s.decided_at = None
    s.decided_by = None
    s.applied_value = None
    await db.commit()
    return {"status": "reverted"}


@router.get(
    "/suggestions/{suggestion_id}"
)  # response_model=L2ASuggestionItem when field alignment is complete
async def get_suggestion(
    suggestion_id: str,
    include: Optional[str] = Query(
        None,
        description="Comma-separated list of additional fields (e.g., 'explanations')",
    ),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),  # Add gateway
):
    """Get a single L2A suggestion by ID with optional detailed information."""
    # Get the suggestion and verify it belongs to this account
    result = await db.execute(
        select(L2ASuggestion)
        .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
        .where(
            L2ASuggestion.id == suggestion_id, L2APolicy.account_id == str(account_id)
        )
    )
    suggestion = result.scalar_one_or_none()

    if not suggestion:
        raise HTTPException(status_code=404, detail="Suggestion not found")

    # Build response
    item = {
        "id": suggestion.id,
        "lead_id": suggestion.lead_id,
        "account_id": suggestion.account_id,
        "score": suggestion.score,
        "tier": suggestion.tier,
        "reasons": suggestion.reasons,
        "explanation": suggestion.explanation,
        "status": suggestion.status,
        "created_at": suggestion.created_at,
    }

    # Add display fields from Salesforce
    # SF gateway is injected via dependency - no need to create client
    if sf:  # Only proceed if gateway is available
        # Gateway handles token persistence automatically

        # Fetch lead details
        try:
            lead = await sf.get_record(
                "Lead",
                suggestion.lead_id,
                fields=["Company", "Email", "Phone", "City", "State"],
            )
            item.update(
                {
                    "lead_company": lead.get("Company"),
                    "lead_email": lead.get("Email"),
                    "lead_phone": lead.get("Phone"),
                }
            )
        except SalesforceConnectionError:
            pass  # Ignore SF connection errors, these are optional fields
        except Exception as e:
            logger.debug(f"Error fetching lead details: {e}")
            pass  # Ignore other errors, these are optional fields

        # Fetch account details
        try:
            account = await sf.get_record(
                "Account",
                suggestion.account_id,
                fields=["Name", "Website", "Phone", "BillingCity", "BillingState"],
            )
            item.update(
                {
                    "account_name": account.get("Name"),
                    "account_website": account.get("Website"),
                    "account_phone": account.get("Phone"),
                    "account_city": account.get("BillingCity"),
                    "account_state": account.get("BillingState"),
                }
            )
        except SalesforceConnectionError:
            pass  # Ignore SF connection errors, these are optional
        except Exception as e:
            logger.debug(f"Error fetching account details: {e}")
            pass  # Ignore other errors, these are optional fields

    # Add ensemble signals if requested
    if include and "explanations" in include:
        if os.getenv("ENSEMBLE_SIGNALS_ENABLED", "true").lower() == "true":
            # Get policy to check multi-algo setting
            pol_res = await db.execute(
                select(L2APolicy).where(L2APolicy.id == suggestion.policy_id)
            )
            policy = pol_res.scalar_one_or_none()
            use_multi_algo = bool(policy and getattr(policy, "use_multi_algo", False))

            # Get field score details from reasons
            fsd = []
            if suggestion.reasons:
                for field_pair, score in suggestion.reasons.items():
                    if "->" in field_pair:
                        src, ref = field_pair.split("->", 1)
                    else:
                        src, ref = field_pair, field_pair

                    fsd.append(
                        {
                            "source_column": src,
                            "reference_column": ref,
                            "score": score,
                            "weight": 1.0,
                            "algorithm": "WRatio",
                        }
                    )

            # Add evidence summary
            summary = summarize_evidence(fsd, use_multi_algo=use_multi_algo)
            item.update(summary)

            # Add field score details and explanations
            item["field_score_details"] = fsd
            item["explanations"] = compact_explanations(fsd)

    return item


@router.get("/metrics")
async def get_l2a_metrics(
    db: AsyncSession = Depends(get_session), account_id: str = Depends(require_account)
):
    """
    Get L2A performance metrics.

    Returns metrics including:
    - Total suggestions generated
    - Number linked, rejected, pending
    - Acceptance rate
    - Last run timestamp
    """
    from sqlalchemy import func
    from sqlalchemy.exc import OperationalError

    acct = str(account_id)

    # Initialize default metrics
    default_metrics = {
        "pending": 0,
        "linked": 0,
        "rejected": 0,
        "total": 0,
        "acceptance_rate": 0.0,
        "last_run_at": None,
        "upgrades_suggested": 0,
        "upgrades_applied": 0,
        "average_gain": 0.0,
        "upgrade_acceptance_rate": 0.0,
    }

    try:
        # Count by status scoped to this account via policy join
        pending = (
            await db.scalar(
                select(func.count())
                .select_from(L2ASuggestion)
                .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
                .where(L2APolicy.account_id == acct, L2ASuggestion.status == "pending")
            )
            or 0
        )

        linked = (
            await db.scalar(
                select(func.count())
                .select_from(L2ASuggestion)
                .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
                .where(L2APolicy.account_id == acct, L2ASuggestion.status == "linked")
            )
            or 0
        )

        rejected = (
            await db.scalar(
                select(func.count())
                .select_from(L2ASuggestion)
                .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
                .where(L2APolicy.account_id == acct, L2ASuggestion.status == "rejected")
            )
            or 0
        )

        last_run_at = await db.scalar(
            select(func.max(L2ASuggestion.created_at))
            .select_from(L2ASuggestion)
            .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
            .where(L2APolicy.account_id == acct)
        )
    except OperationalError as e:
        # If basic queries fail, return defaults
        logger.warning(f"L2A metrics basic queries failed: {e}")
        return {"metrics": default_metrics}

    # Add upgrade metrics (with fallback for missing columns)
    try:
        upgrades_suggested = (
            await db.scalar(
                select(func.count())
                .select_from(L2ASuggestion)
                .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
                .where(
                    L2APolicy.account_id == acct,
                    L2ASuggestion.reason_code == "upgrade",
                    L2ASuggestion.status == "pending",
                )
            )
            or 0
        )
    except OperationalError as e:
        if "no such column" in str(e):
            upgrades_suggested = 0
        else:
            raise

    try:
        upgrades_applied = (
            await db.scalar(
                select(func.count())
                .select_from(L2ASuggestion)
                .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
                .where(
                    L2APolicy.account_id == acct,
                    L2ASuggestion.reason_code == "upgrade",
                    L2ASuggestion.status == "linked",
                )
            )
            or 0
        )
    except OperationalError as e:
        if "no such column" in str(e):
            upgrades_applied = 0
        else:
            raise

    try:
        avg_gain = await db.scalar(
            select(func.avg(L2ASuggestion.gain))
            .select_from(L2ASuggestion)
            .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
            .where(L2APolicy.account_id == acct, L2ASuggestion.reason_code == "upgrade")
        )
    except OperationalError as e:
        if "no such column" in str(e):
            avg_gain = None
        else:
            raise

    decisions = linked + rejected
    accept_rate = (linked / decisions) if decisions > 0 else None

    # POLISH: 30-day metrics for better trends
    from datetime import timedelta

    thirty_days_ago = datetime.utcnow() - timedelta(days=30)

    try:
        recent_gains_result = await db.execute(
            select(L2ASuggestion.gain)
            .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
            .where(
                L2APolicy.account_id == account_id,
                L2ASuggestion.reason_code == "upgrade",
                L2ASuggestion.created_at >= thirty_days_ago,
                L2ASuggestion.gain.isnot(None),
            )
        )
        gains = [g for (g,) in recent_gains_result.all()]
    except OperationalError:
        gains = []

    if gains:
        avg_gain_30d = sum(gains) / len(gains)
        median_gain_30d = sorted(gains)[len(gains) // 2]
        max_gain_30d = max(gains)
        min_gain_30d = min(gains)
    else:
        avg_gain_30d = None
        median_gain_30d = None
        max_gain_30d = None
        min_gain_30d = None

    # Count 30-day activity
    try:
        recent_activity = await db.execute(
            select(L2ASuggestion.status, func.count(L2ASuggestion.id))
            .join(L2APolicy, L2APolicy.id == L2ASuggestion.policy_id)
            .where(
                L2APolicy.account_id == account_id,
                L2ASuggestion.created_at >= thirty_days_ago,
            )
            .group_by(L2ASuggestion.status)
        )

        activity_30d = {}
        for status, count in recent_activity.all():
            activity_30d[status] = count
    except OperationalError:
        activity_30d = {}

    return {
        "metrics": {
            "pending": pending,
            "applied": linked,  # alias for UI
            "linked": linked,  # BC
            "rejected": rejected,
            "accept_rate": accept_rate,
            "acceptance_rate": accept_rate,  # BC
            "last_run_at": last_run_at.isoformat() if last_run_at else None,
            "upgrades_suggested": upgrades_suggested,
            "upgrades_applied": upgrades_applied,
            "avg_gain": float(avg_gain) if avg_gain else None,
            # POLISH: Enhanced 30-day metrics
            "avg_gain_30d": avg_gain_30d,
            "median_gain_30d": median_gain_30d,
            "max_gain_30d": max_gain_30d,
            "min_gain_30d": min_gain_30d,
            "upgrades_30d": len(gains),
            "activity_30d": activity_30d,
            "total_30d": sum(activity_30d.values()),
        }
    }


# ==========================
# Saved Views (Manual Studio)
# ==========================


@router.get("/manual/views")
async def list_manual_views(
    object_type: str = Query(..., pattern=r"^(Lead|Account)$"),
    scope: str = Query("user", pattern=r"^(user|org)$"),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    base = select(L2AManualView).where(
        L2AManualView.account_id == str(account_id),
        L2AManualView.object_type == object_type,
    )
    if scope == "org":
        q = base.where(L2AManualView.scope == "org").order_by(
            L2AManualView.updated_at.desc()
        )
    else:
        q = base.where(
            (L2AManualView.scope == "org")
            | (
                (L2AManualView.scope == "user")
                & (L2AManualView.user_id == str(user_id))
            )
        ).order_by(L2AManualView.updated_at.desc())
    rows = (await db.execute(q)).scalars().all()
    return [
        {
            "id": r.id,
            "name": r.name,
            "object_type": r.object_type,
            "filters": _coerce_list(r.filters_json),
            "scope": getattr(r, "scope", "user"),
            "updated_at": (r.updated_at or _dt.datetime.utcnow()).isoformat(),
        }
        for r in rows
    ]


@router.post("/manual/views")
async def save_manual_view(
    body: SavedViewCreate,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    # Upsert by (account_id, user_id, object_type, name)
    q = select(L2AManualView).where(
        L2AManualView.account_id == str(account_id),
        L2AManualView.object_type == body.object_type,
        L2AManualView.name == body.name,
        L2AManualView.scope == (body.scope or "user"),
        (L2AManualView.user_id == str(user_id))
        if (body.scope or "user") == "user"
        else (L2AManualView.user_id == L2AManualView.user_id),
    )
    existing = (await db.execute(q)).scalar_one_or_none()
    if existing:
        existing.filters_json = json.dumps(body.filters or [])
        await db.commit()
        return {"id": existing.id, "updated": True}

    row = L2AManualView(
        account_id=str(account_id),
        user_id=str(user_id),
        object_type=body.object_type,
        name=body.name,
        filters_json=json.dumps(body.filters or []),
        scope=(body.scope or "user"),
    )
    db.add(row)
    await db.commit()
    return {"id": row.id, "created": True}


@router.patch("/manual/views/{view_id}")
async def update_manual_view(
    view_id: str,
    body: SavedViewUpdate,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    q = select(L2AManualView).where(
        L2AManualView.id == view_id,
        L2AManualView.account_id == str(account_id),
        L2AManualView.user_id == str(user_id),
    )
    row = (await db.execute(q)).scalar_one_or_none()
    if not row:
        raise HTTPException(404, "View not found")
    if body.name is not None:
        row.name = body.name
    if body.filters is not None:
        row.filters_json = json.dumps(body.filters or [])
    await db.commit()
    return {"id": view_id, "updated": True}


@router.delete("/manual/views/{view_id}")
async def delete_manual_view(
    view_id: str,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    user_id: str = Depends(require_user),
):
    q = delete(L2AManualView).where(
        L2AManualView.id == view_id,
        L2AManualView.account_id == str(account_id),
        L2AManualView.user_id == str(user_id),
    )
    await db.execute(q)
    await db.commit()
    return {"deleted": True}


# ========================
# Company Hierarchy (Acct)
# ========================


class HierarchyOut(BaseModel):
    ultimate_parent: Optional[Dict[str, Any]] = None
    lineage: List[Dict[str, Any]] = []  # chain to parent
    children: List[Dict[str, Any]] = []  # direct children


@router.get("/company/hierarchy")
async def company_hierarchy(
    account_id: str = Query(...),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
    # Ensure this route is protected and scoped to an authenticated account/user
    _account_id: str = Depends(require_account),
    _user_id: str = Depends(require_user),
):
    # 1) fetch this account (Id, Name, ParentId)
    me = (
        await sf.get_record("Account", account_id, fields=["Id", "Name", "ParentId"])
    ) or {}
    if not me:
        raise HTTPException(404, "Account not found")

    # 2) climb to ultimate parent (follow ParentId chain up to 8 hops)
    lineage: List[Dict[str, Any]] = []
    cur = me
    for _ in range(8):
        pid = cur.get("ParentId")
        if not pid:
            break
        parent = await sf.get_record("Account", pid, fields=["Id", "Name", "ParentId"])
        if not parent:
            break
        lineage.append(parent)
        cur = parent
    ultimate = cur

    # 3) fetch children of this account
    q = f"SELECT Id, Name FROM Account WHERE ParentId = '{me['Id']}' LIMIT 200"
    res = await sf.soql(q)
    kids = res.get("records", []) if isinstance(res, dict) else []
    return {"ultimate_parent": ultimate, "lineage": lineage, "children": kids}


# ====================
# Recently Applied L2A
# ====================


@router.get("/recently_applied")
async def recently_applied(
    limit: int = 10,
    hydrate: bool = True,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(require_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    pol_ids = select(L2APolicy.id).where(
        L2APolicy.account_id == str(account_id), L2APolicy.status == "ACTIVE"
    )
    q = (
        select(L2ASuggestion)
        .where(
            L2ASuggestion.policy_id.in_(pol_ids),
            L2ASuggestion.status == "linked",
        )
        .order_by(L2ASuggestion.decided_at.desc())
        .limit(max(1, min(int(limit or 10), 50)))
    )
    rows = (await db.execute(q)).scalars().all()
    out = []

    # Batch fetch all Salesforce data if hydrating
    lead_map = {}
    account_map = {}

    if hydrate and sf and rows:
        logger.info(f"Fetching Salesforce data for {len(rows)} recently applied items")
        # Collect unique IDs
        lead_ids = list(set(r.lead_id for r in rows if r.lead_id))
        account_ids = list(
            set(
                r.applied_value or r.account_id
                for r in rows
                if (r.applied_value or r.account_id)
            )
        )

        # Batch fetch leads (Salesforce supports up to 2000 IDs in a single query)
        if lead_ids:
            try:
                lead_ids_clause = ",".join(f"'{lid}'" for lid in lead_ids[:200])
                lead_query = f"SELECT Id, Company FROM Lead WHERE Id IN ({lead_ids_clause}) LIMIT 200"
                leads_result = await sf.soql(lead_query)
                logger.info(
                    f"Lead query result: {len(leads_result.get('records', []))} records"
                )
                if leads_result and leads_result.get("records"):
                    for lead in leads_result["records"]:
                        lead_map[lead["Id"]] = lead
            except Exception as e:
                logger.error(f"Could not batch fetch leads: {e}")

        # Batch fetch accounts
        if account_ids:
            try:
                account_ids_clause = ",".join(f"'{aid}'" for aid in account_ids[:200])
                account_query = f"SELECT Id, Name FROM Account WHERE Id IN ({account_ids_clause}) LIMIT 200"
                accounts_result = await sf.soql(account_query)
                if accounts_result and accounts_result.get("records"):
                    for account in accounts_result["records"]:
                        account_map[account["Id"]] = account
            except Exception as e:
                logger.error(f"Could not batch fetch accounts: {e}")

    # Build response items
    for r in rows:
        decided = r.decided_at or r.created_at or _dt.datetime.utcnow()
        item = {
            "id": r.id,
            "lead_id": r.lead_id,
            "account_id": r.account_id,
            "applied_field": r.applied_field,
            "previous_value": r.previous_value,
            "applied_value": r.applied_value,
            "updated_at": decided.isoformat(),
            "undo_deadline": (decided + timedelta(hours=24)).isoformat(),
        }

        # Add score and tier (will be None if not set, which is fine)
        item["score"] = getattr(r, "score", None)
        item["tier"] = getattr(r, "tier", None)

        # Set name fields from batch-fetched data or use defaults
        lead = lead_map.get(r.lead_id, {})
        item["lead_company"] = (
            lead.get("Company") or lead.get("Name") or f"Lead {r.lead_id[:10]}"
        )
        item["lead_name"] = lead.get("Name", "")
        item["_deleted"] = not bool(lead)

        account_id_to_fetch = r.applied_value or r.account_id
        account = account_map.get(account_id_to_fetch, {})
        item["account_name"] = (
            account.get("Name") or f"Account {account_id_to_fetch[:10]}"
        )

        out.append(item)
    return out
