from __future__ import annotations
import asyncio
import json
import uuid
import logging
import os
import hashlib
import re
from datetime import datetime
import datetime as _dt
from typing import Optional, List, Literal, Dict, Any, Set
from collections import defaultdict
import time

from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel, field_validator, Field, AliasChoices
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text, select, func
from sqlalchemy.exc import OperationalError

from ...db import get_session
from ...auth_core import get_current_account
from ...services.reporting_emitter import EventEmitter
from ...model_defs.reporting_models import ChangeSet
from ...services.salesforce_gateway import SalesforceGateway, get_salesforce_gateway
from ...services.dedupe_service_v3 import PolicyDrivenDedupeService
from ...model_defs.dedupe_models import DedupePolicy
from ...services.explain_utils import classify_match_type
from ...services.ensemble_explanations import compact_explanations
from ...services.dedupe_service_intelligent import ensure_dedupe_suggestions_schema
from .schemas import DedupeSuggestionsResponse, MergePlanResponse
try:
    from ...services.run_locks import get_lock
except ImportError as exc:  # pragma: no cover
    logging.getLogger(__name__).warning("run_locks module unavailable: %s", exc)

    def get_lock(*_args, **_kwargs):  # type: ignore
        return asyncio.Lock()
from ...utils.logging_utils import _s

router = APIRouter(prefix="/api/v2/dedupe", tags=["dedupe"])
log = logging.getLogger(__name__)


# Feature flags
def _env_bool(name: str, default: bool = False) -> bool:
    v = os.getenv(name)
    if v is None:
        return default
    return str(v).lower() in ("1", "true", "yes", "y", "on")


ENSEMBLE_SIGNALS_ENABLED = _env_bool("ENSEMBLE_SIGNALS_ENABLED", True)
ENSEMBLE_EXPLANATION_MAX = int(os.getenv("ENSEMBLE_EXPLANATION_MAX", "5") or 5)
ENSEMBLE_FLAG_MAX = int(os.getenv("ENSEMBLE_FLAG_MAX", "4") or 4)
from ...utils.score_utils import (
    norm01 as _norm01,
)

ENSEMBLE_CONTRIB_THRESHOLD = float(
    os.getenv("ENSEMBLE_CONTRIB_THRESHOLD", "0.10") or 0.10
)


def _parse_details(details_text: str | None) -> list[dict]:
    if not details_text:
        return []
    try:
        data = json.loads(details_text)
        return data if isinstance(data, list) else []
    except Exception:
        return []


def _evidence_aggregate(details: list[dict]) -> dict:
    """Compute evidence_count/strength/flags consistently with L2A."""
    if not details:
        return {
            "evidence_count": 0,
            "evidence_strength": 0.0,
            "evidence_flags": [],
            "ensemble_used": False,
        }

    contribs = []
    for d in details:
        src = d.get("source_column") or d.get("source_field") or "?"
        ref = d.get("reference_column") or d.get("ref_field") or src
        pair = f"{src}->{ref}"
        w = float(d.get("weight", 1.0))
        s = _norm01(d.get("score", 0.0))
        alg = d.get("algorithm", "Unknown")
        contribs.append((w, s, alg, pair))

    # Sort by weighted contribution for flags
    scored = [(pair, w * s, alg) for w, s, alg, pair in contribs]
    scored.sort(key=lambda t: t[1], reverse=True)
    flags = [
        p
        for p, val, alg in scored[:ENSEMBLE_FLAG_MAX]
        if val >= ENSEMBLE_CONTRIB_THRESHOLD
    ]

    # Proper weighted average for strength (0..1)
    total_weight = sum(w for w, _, _, _ in contribs) or 1.0
    strength = sum(w * s for w, s, _, _ in contribs) / total_weight

    # Ensemble considered "used" if multiple non-Exact algos or flag enabled and we have details
    non_exact = any((alg or "").lower() != "exact" for _, _, alg in scored)
    return {
        "evidence_count": len(details),
        "evidence_strength": round(strength, 3),
        "evidence_flags": flags,
        "ensemble_used": bool(
            ENSEMBLE_SIGNALS_ENABLED and (non_exact or len(details) > 1)
        ),
    }


def _reasons_from_details(details: list[dict]) -> str:
    """Build reasons JSON from details list."""
    if not details:
        return "{}"
    items: Dict[str, float] = {}
    for d in details:
        src = d.get("source_column") or d.get("source_field") or "?"
        ref = d.get("reference_column") or d.get("ref_field") or src
        key = f"{src}->{ref}"
        val = round(_norm01(d.get("score", 0.0)) * 100.0, 1)
        items[key] = max(items.get(key, 0.0), val)
    return json.dumps(items)


# --------- helpers ----------
async def ensure_dedupe_schema(db: AsyncSession):
    """Ensure dedupe_suggestions table exists with proper schema."""
    await ensure_dedupe_suggestions_schema(db)


def _build_crm_dedupe_runner(
    db: AsyncSession, account_id: str, backend_mode: str
) -> tuple[Any, str]:
    if backend_mode == "intelligent" and _env_bool("DEDUPE_V2_ENABLED", True):
        from ...services import dedupe_service_intelligent_v2 as dedupe_v2

        return dedupe_v2.IntelligentDedupeOrchestrator(db, account_id), "v2"
    return PolicyDrivenDedupeService(db, account_id), "policy_v3"


# Helper function removed - using gateway dependency injection now
# All functions now use: sf: SalesforceGateway = Depends(get_salesforce_gateway)


# --------- request models ----------
class DedupeRunRequest(BaseModel):
    object_type: Literal["Lead", "Account", "Contact"]
    threshold_override: Optional[float] = None
    max_records: Optional[int] = 5000
    use_multi_algo: Optional[bool] = Field(
        default=None, validation_alias=AliasChoices("use_ensemble", "use_multi_algo")
    )
    apply_blocking: Optional[bool] = True  # reuse engine blocking
    mode: Literal["intelligent", "exact", "incremental", "rebuild"] = "intelligent"
    strong_signals_only: Optional[bool] = (
        False  # Filter to strong identity anchors only
    )

    @field_validator("threshold_override")
    @classmethod
    def v_thresh(cls, v):
        if v is None:
            return v
        v = float(v)
        if not (0.0 <= v <= 1.0):
            raise ValueError("threshold_override must be between 0.0 and 1.0")
        return v


# --------- endpoints ----------


class DedupePreflightResponse(BaseModel):
    """Preflight validation response"""

    records_count: int
    has_email: bool
    has_phone: bool
    has_company: bool
    has_website: bool
    has_name: bool
    email_domain_overlap: int
    name_prefix_overlap: int
    blocking_strategy: str
    recommended_threshold: float
    warnings: list[str] = []


@router.post("/preflight")
async def dedupe_preflight(
    payload: DedupeRunRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
) -> DedupePreflightResponse:
    """
    Pre-run validation that checks data quality and blocking feasibility.
    Returns info about what fields are available and recommended settings.
    """
    import pandas as pd

    service, _runner_name = _build_crm_dedupe_runner(db, account_id, "intelligent")

    try:
        # Fetch a sample of records
        sample_limit = min(payload.limit or 1000, 1000)
        recs = await service.fetch_records(sf, payload.object_type, limit=sample_limit)

        if not recs:
            return DedupePreflightResponse(
                records_count=0,
                has_email=False,
                has_phone=False,
                has_company=False,
                has_website=False,
                has_name=False,
                email_domain_overlap=0,
                name_prefix_overlap=0,
                blocking_strategy="none",
                recommended_threshold=0.70,
                warnings=["No records found in Salesforce"],
            )

        # Check field availability
        df = pd.DataFrame(recs)

        # Normalize object type once
        kind = str(payload.object_type).split(".")[-1].lower()

        has_email = "Email" in df.columns and df["Email"].notna().sum() > 0
        has_phone = any(
            col in df.columns and df[col].notna().sum() > 0
            for col in ["Phone", "MobilePhone"]
        )
        has_company = "Company" in df.columns and df["Company"].notna().sum() > 0
        has_website = "Website" in df.columns and df["Website"].notna().sum() > 0
        has_name = any(
            col in df.columns and df[col].notna().sum() > 0
            for col in ["Name", "FirstName", "LastName"]
        )

        # Calculate overlaps for blocking strategy
        email_domain_overlap = 0
        name_prefix_overlap = 0
        blocking_strategy = "none"
        recommended_threshold = 0.70
        warnings = []

        # Object-specific blocking strategy
        if kind == "lead":
            if has_email:
                # Check for repeated emails
                email_lc = df["Email"].astype(str).str.lower()
                email_repeats = (email_lc.value_counts() > 1).sum()
                if email_repeats > 0:
                    blocking_strategy = "email"
                    recommended_threshold = 0.72
                    email_domain_overlap = int(email_repeats)

            if blocking_strategy == "none" and has_company:
                # Check company name prefix overlap
                company_norm = df["Company"].dropna().str.lower().str[:8]
                prefix_repeats = (company_norm.value_counts() > 1).sum()
                if prefix_repeats > 0:
                    blocking_strategy = "name(k=8)"
                    recommended_threshold = 0.72
                    name_prefix_overlap = int(prefix_repeats)
                else:
                    # Try wider prefix
                    company_norm = df["Company"].dropna().str.lower().str[:4]
                    prefix_repeats = (company_norm.value_counts() > 1).sum()
                    if prefix_repeats > 0:
                        blocking_strategy = "name(k=4)"
                        recommended_threshold = 0.72
                        name_prefix_overlap = int(prefix_repeats)
                    else:
                        blocking_strategy = "none"
                        recommended_threshold = 0.66

        elif kind == "contact":
            if has_email:
                # Check for repeated emails
                email_lc = df["Email"].astype(str).str.lower()
                email_repeats = (email_lc.value_counts() > 1).sum()
                if email_repeats > 0:
                    blocking_strategy = "email"
                    recommended_threshold = 0.72
                    email_domain_overlap = int(email_repeats)

            if blocking_strategy == "none" and (
                "LastName" in df.columns or "FirstName" in df.columns
            ):
                # Try last name prefix
                if "LastName" in df.columns:
                    name_col = "LastName"
                elif "FirstName" in df.columns:
                    name_col = "FirstName"
                else:
                    name_col = None

                if name_col:
                    name_norm = df[name_col].dropna().str.lower().str[:8]
                    prefix_repeats = (name_norm.value_counts() > 1).sum()
                    if prefix_repeats > 0:
                        blocking_strategy = "name(k=8)"
                        recommended_threshold = 0.72
                        name_prefix_overlap = int(prefix_repeats)
                    else:
                        blocking_strategy = "none"
                        recommended_threshold = 0.66

        elif kind == "account":
            if has_website:
                # Check for repeated domains
                website_domains = df["Website"].dropna().apply(
                    PolicyDrivenDedupeService.norm_domain
                )
                domain_repeats = (website_domains.value_counts() > 1).sum()
                if domain_repeats > 0:
                    blocking_strategy = "domain"
                    recommended_threshold = 0.72
                    email_domain_overlap = int(
                        domain_repeats
                    )  # Reuse field for domain overlap

            if blocking_strategy == "none" and has_name:
                # Check Name prefix overlap
                name_norm = df["Name"].dropna().str.lower().str[:8]
                prefix_repeats = (name_norm.value_counts() > 1).sum()
                if prefix_repeats > 0:
                    blocking_strategy = "name(k=8)"
                    recommended_threshold = 0.72
                    name_prefix_overlap = int(prefix_repeats)
                else:
                    # Try wider prefix
                    name_norm = df["Name"].dropna().str.lower().str[:4]
                    prefix_repeats = (name_norm.value_counts() > 1).sum()
                    if prefix_repeats > 0:
                        blocking_strategy = "name(k=4)"
                        recommended_threshold = 0.72
                        name_prefix_overlap = int(prefix_repeats)
                    else:
                        blocking_strategy = "none"
                        recommended_threshold = 0.66

        # Generate warnings
        if not has_email and not has_phone:
            warnings.append(
                "No email or phone fields found - matching quality may be limited"
            )

        if kind == "lead" and not has_company:
            warnings.append(
                "No Company field found for Leads - fuzzy matching will be limited"
            )

        if kind == "account" and not has_website:
            warnings.append(
                "No Website field found for Accounts - domain matching unavailable"
            )

        if blocking_strategy == "none":
            warnings.append(
                "No effective blocking strategy found - processing may be slower"
            )

        return DedupePreflightResponse(
            records_count=len(recs),
            has_email=has_email,
            has_phone=has_phone,
            has_company=has_company,
            has_website=has_website,
            has_name=has_name,
            email_domain_overlap=email_domain_overlap,
            name_prefix_overlap=name_prefix_overlap,
            blocking_strategy=blocking_strategy,
            recommended_threshold=recommended_threshold,
            warnings=warnings,
        )

    except Exception as e:
        log.error(f"Preflight check failed: {e}")
        return DedupePreflightResponse(
            records_count=0,
            has_email=False,
            has_phone=False,
            has_company=False,
            has_website=False,
            has_name=False,
            email_domain_overlap=0,
            name_prefix_overlap=0,
            blocking_strategy="error",
            recommended_threshold=0.70,
            warnings=[f"Preflight check failed: {str(e)}"],
        )


@router.post("/run")
async def run_dedupe(
    payload: DedupeRunRequest,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """
    Kick off a dedupe pass for Lead/Account/Contact.
    Phase 2: Uses Intelligence + Engine when available, falls back to exact matching.
    """
    # Normalize object type and get lock
    kind = str(payload.object_type).split(".")[-1].lower()
    lock = get_lock(account_id, kind)

    log.info(_s(f"[DEDUPE] Lock wait account={account_id} object={kind}"))
    async with lock:
        log.info(_s(f"[DEDUPE] Lock acquired account={account_id} object={kind}"))

        # Handle rebuild mode - clear existing suggestions BUT preserve manual ones
        if payload.mode == "rebuild":
            # Count before delete for logging
            count_result = await db.execute(
                text("""
                SELECT status, COUNT(*) as cnt
                FROM dedupe_suggestions
                WHERE account_id = :account_id
                AND object_type = :object_type
                AND status IN ('pending', 'proposed', 'new')
                GROUP BY status
            """),
                {"account_id": str(account_id), "object_type": payload.object_type},
            )
            counts_before = {row.status: row.cnt for row in count_result}

            # Delete pending, proposed, and new suggestions EXCEPT manual ones
            # Manual clusters have "Manual" in their reasons field
            delete_result = await db.execute(
                text("""
                DELETE FROM dedupe_suggestions
                WHERE account_id = :account_id
                AND object_type = :object_type
                AND status IN ('pending', 'proposed', 'new')
                AND reasons NOT LIKE '%Manual%'
            """),
                {"account_id": str(account_id), "object_type": payload.object_type},
            )
            await db.commit()

            log.info(
                f"Rebuild mode: deleted {delete_result.rowcount} auto-generated suggestions (preserved manual). Counts by status: {counts_before}"
            )

        # Map UI modes to backend modes
        backend_mode = (
            "intelligent"
            if payload.mode in ["incremental", "rebuild"]
            else payload.mode
        )

        svc, runner_name = _build_crm_dedupe_runner(db, account_id, backend_mode)

        # SF gateway is injected via dependency - no need to create client

        # Run dedupe through orchestrator - now returns meta too
        save_integration = None
        try:
            fetched, saved, meta = await svc.run_on_salesforce(
                sf=sf,
                obj=payload.object_type,
                limit=payload.max_records or 5000,
                mode=backend_mode,
                threshold_override=payload.threshold_override,
                use_multi_algo=payload.use_multi_algo,
                save_integration=save_integration,
            )

            # Check for failure indicators
            if (
                saved == 0 and fetched > 10
            ):  # No results with substantial data is suspicious
                log.warning(
                    f"Dedupe produced 0 results from {fetched} records - possible engine failure"
                )
                # Check meta for more details
                if meta.get("error"):
                    log.error(f"Dedupe engine error: {meta['error']}")
                    raise HTTPException(
                        500,
                        f"Dedupe engine failed: {meta.get('error', 'No duplicates found or engine error')}",
                    )

        except HTTPException:
            raise  # Re-raise HTTP exceptions
        except Exception as e:
            log.error(f"Dedupe run failed: {e}", exc_info=True)
            raise HTTPException(500, f"Dedupe processing failed: {str(e)}")

        log.info(_s(f"[DEDUPE] Run complete account={account_id} object={kind}"))
        log.info(
            "Dedupe run runner=%s: obj=%s mode=%s fetched=%s saved=%s",
            runner_name,
            payload.object_type,
            payload.mode,
            fetched,
            saved,
        )

        # Ensure schema exists but don't add fake seed data
        await ensure_dedupe_schema(db)

        # Build response body inside the lock context
        if saved == 0:
            log.warning(
                "Dedupe run completed with zero suggestions: object=%s mode=%s fetched=%s",
                payload.object_type,
                payload.mode,
                fetched,
            )
        body = {
            "object": payload.object_type,
            "fetched": fetched,
            "suggestions_generated": saved,
            "mode": payload.mode,
            "threshold_used": payload.threshold_override or 0.60,
            "policy_version": meta.get("policy_version", "unknown"),
            "anchors_applied": meta.get("anchors_applied", []),
            "rules_applied": meta.get("rules_applied", []),
            "ensemble_summary": {
                "used": bool(meta.get("ensemble_used", False)),
                "present_only_scoring": True,
            },
        }

    # Lock released when we exit the async context
    log.info(_s(f"[DEDUPE] Lock released account={account_id} object={kind}"))
    return body


@router.get(
    "/suggestions",
    response_model=DedupeSuggestionsResponse,
    response_model_exclude_none=True,
)
async def list_suggestions(
    object_type: Optional[Literal["Lead", "Contact", "Account"]] = Query(None),
    status: Optional[Literal["pending", "accepted", "rejected", "all"]] = "pending",
    min_score: Optional[float] = 0.0,
    q: str | None = Query(None, description="Search query for reasons and IDs"),
    sort_by: str | None = Query(None, description="Sort by: score, tier"),
    sort_dir: str | None = Query(None, description="Sort direction: asc, desc"),
    page: int = 1,
    page_size: int = 25,
    include: Optional[str] = None,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    include_expl = include and "explanations" in include

    where = ["account_id = :account_id"]
    params: Dict[str, Any] = {"account_id": account_id}
    if object_type:
        where.append("object_type = :obj")
        params["obj"] = object_type
    if status and status != "all":
        where.append("status = :st")
        params["st"] = status
    if min_score is not None:
        where.append("score >= :minscore")
        params["minscore"] = float(min_score)
    if q:
        where.append(
            "(reasons LIKE :search OR rec_id_1 LIKE :search OR rec_id_2 LIKE :search)"
        )
        params["search"] = f"%{q}%"

    where_sql = " AND ".join(where)

    # Sort handling
    sort_col = "score"  # default
    if sort_by and sort_by.lower() in ["score", "tier"]:
        sort_col = sort_by.lower()
    sort_direction = "DESC" if (sort_dir or "desc").lower() == "desc" else "ASC"

    limit = int(page_size or 25)
    offset = max(0, (int(page or 1) - 1) * limit)

    # Get total count for pagination
    count_sql = text(f"SELECT COUNT(*) FROM dedupe_suggestions WHERE {where_sql}")
    total = (await db.execute(count_sql, params)).scalar_one() or 0

    sql = text(f"""
      SELECT id, object_type, rec_id_1, rec_id_2, score, tier,
             reasons, field_score_details, explanation, status, created_at
      FROM dedupe_suggestions
      WHERE {where_sql}
      ORDER BY {sort_col} {sort_direction}, created_at DESC
      LIMIT :limit OFFSET :offset
    """)
    params.update({"limit": limit, "offset": offset})

    rows = (await db.execute(sql, params)).mappings().all()
    items = []
    skipped = 0

    # Define allowed values for normalization
    allowed_obj = {"Lead", "Contact", "Account"}
    allowed_status = {"pending", "accepted", "rejected"}

    for r in rows:
        details = _parse_details(r["field_score_details"])
        agg = _evidence_aggregate(details)

        # reasons: JSON string -> dict (backward compat)
        reasons_dict = {}
        try:
            raw_reasons = r.get("reasons")
            if isinstance(raw_reasons, str):
                reasons_dict = json.loads(raw_reasons) if raw_reasons else {}
            elif isinstance(raw_reasons, dict):
                reasons_dict = raw_reasons
        except Exception:
            reasons_dict = {}

        if not reasons_dict and details:
            for d in details:
                src = d.get("source_column") or d.get("source_field") or "?"
                ref = d.get("reference_column") or d.get("ref_field") or src
                key = f"{src}->{ref}"
                sc = float(d.get("score") or 0.0)
                if sc > 1.0:
                    sc /= 100.0
                reasons_dict[key] = max(
                    reasons_dict.get(key, 0.0), round(sc * 100.0, 1)
                )

        try:
            obj = r.get("object_type") or object_type or "Account"
            obj = obj if obj in allowed_obj else "Account"
            st = r.get("status")
            st = st if (st in allowed_status) else None
            tier = r.get("tier") or "POSSIBLE"

            item = {
                "id": r["id"],
                "object_type": obj,
                "left_id": r["rec_id_1"],
                "right_id": r["rec_id_2"],
                "score": float(r.get("score") or 0.0),
                "tier": tier,
                "status": st,
                "created_at": r.get("created_at"),
                "reasons": reasons_dict,
                "evidence_count": agg.get("evidence_count", 0),
                "evidence_strength": agg.get("evidence_strength", 0.0),
                "evidence_flags": agg.get("evidence_flags", []),
                "ensemble_used": agg.get("ensemble_used", False),
            }
        except Exception:
            logging.getLogger("fmatch.api").exception(
                "Dedupe list: row mapping failed (id=%s)", r.get("id")
            )
            logging.getLogger("fmatch.api").error(
                "Row keys present: %s", list(r.keys())
            )
            skipped += 1
            # Skip bad legacy row so one record doesn't 500 the whole page
            continue

        # Always match_type
        item["match_type"] = classify_match_type(details)

        if include_expl:
            item["field_score_details"] = details or []
            item["explanations"] = compact_explanations(details or [])

        items.append(item)

    # Log request stats
    logging.getLogger("fmatch.api").info(
        "Dedupe list built: account=%s page=%s size=%s items=%s skipped=%s",
        account_id,
        page,
        page_size,
        len(items),
        skipped,
    )

    # Return the payload directly - FastAPI will handle validation with response_model
    return {"items": items, "page": page, "page_size": page_size, "total": int(total)}


@router.get(
    "/suggestions/{sid}"
)  # response_model=DedupeSuggestionItem when field alignment is complete
async def get_suggestion(
    sid: str,
    include: Optional[str] = None,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    include_expl = include and "explanations" in include
    sql = text("""
      SELECT id, object_type, rec_id_1, rec_id_2, score, tier,
             reasons, field_score_details, explanation, status, created_at
      FROM dedupe_suggestions
      WHERE id = :sid AND account_id = :account_id
      LIMIT 1
    """)
    row = (
        (await db.execute(sql, {"sid": sid, "account_id": account_id}))
        .mappings()
        .first()
    )
    if not row:
        raise HTTPException(status_code=404, detail="Not found")

    details = _parse_details(row["field_score_details"])
    agg = _evidence_aggregate(details)

    # reasons normalization (same as list)
    reasons_dict = {}
    try:
        raw_reasons = row.get("reasons")
        if isinstance(raw_reasons, str):
            reasons_dict = json.loads(raw_reasons) if raw_reasons else {}
        elif isinstance(raw_reasons, dict):
            reasons_dict = raw_reasons
    except Exception:
        reasons_dict = {}
    if not reasons_dict and details:
        for d in details:
            src = d.get("source_column") or d.get("source_field") or "?"
            ref = d.get("reference_column") or d.get("ref_field") or src
            key = f"{src}->{ref}"
            sc = float(d.get("score") or 0.0)
            if sc > 1.0:
                sc /= 100.0
            reasons_dict[key] = max(reasons_dict.get(key, 0.0), round(sc * 100.0, 1))

    obj = row.get("object_type") or "Account"
    if obj not in {"Lead", "Contact", "Account"}:
        obj = "Account"
    st = row.get("status")
    if st not in {"pending", "accepted", "rejected"}:
        st = None
    tier = row.get("tier") or "POSSIBLE"

    item = {
        "id": row["id"],
        "object_type": obj,
        "left_id": row["rec_id_1"],
        "right_id": row["rec_id_2"],
        "score": float(row.get("score") or 0.0),
        "tier": tier,
        "status": st,
        "created_at": row.get("created_at"),
        "reasons": reasons_dict,
        "evidence_count": agg.get("evidence_count", 0),
        "evidence_strength": agg.get("evidence_strength", 0.0),
        "evidence_flags": agg.get("evidence_flags", []),
        "ensemble_used": agg.get("ensemble_used", False),
        "explanation": row.get("explanation"),
    }
    item["match_type"] = classify_match_type(details)
    if include_expl:
        item["field_score_details"] = details or []
        item["explanations"] = compact_explanations(details or [])

    return item


@router.get("/metrics")
async def get_dedupe_metrics(
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    """
    Get dedupe performance metrics.

    Returns metrics including:
    - Total suggestions generated
    - Number accepted, rejected, pending
    - Acceptance rate
    - Number of clusters pending review
    - Number of pairs merged
    """

    # Ensure the dedupe_suggestions table has the correct schema
    await ensure_dedupe_schema(db)

    # Initialize default metrics
    default_metrics = {
        "pending": 0,
        "accepted": 0,
        "rejected": 0,
        "total": 0,
        "acceptance_rate": 0.0,
        "clusters_pending": 0,
        "pairs_merged": 0,
        "last_run_at": None,
    }

    try:
        # Count by status
        pending = (
            await db.scalar(
                select(func.count())
                .select_from(text("dedupe_suggestions"))
                .where(text("account_id = :account_id AND status = 'pending'"))
                .params(account_id=account_id)
            )
            or 0
        )

        accepted = (
            await db.scalar(
                select(func.count())
                .select_from(text("dedupe_suggestions"))
                .where(text("account_id = :account_id AND status = 'accepted'"))
                .params(account_id=account_id)
            )
            or 0
        )

        rejected = (
            await db.scalar(
                select(func.count())
                .select_from(text("dedupe_suggestions"))
                .where(text("account_id = :account_id AND status = 'rejected'"))
                .params(account_id=account_id)
            )
            or 0
        )

        total = pending + accepted + rejected

        # Calculate acceptance rate
        decided = accepted + rejected
        acceptance_rate = (accepted / decided * 100.0) if decided > 0 else 0.0

        # Get clusters pending (unique cluster IDs with pending suggestions)
        # Note: cluster_id column is guaranteed to exist after ensure_dedupe_schema()
        clusters_result = await db.execute(
            text("""
                SELECT COUNT(DISTINCT cluster_id) as count
                FROM dedupe_suggestions
                WHERE account_id = :account_id
                AND status = 'pending'
                AND cluster_id IS NOT NULL
            """),
            {"account_id": account_id},
        )
        clusters_pending = clusters_result.scalar() or 0

        # Get last run timestamp
        last_run_result = await db.execute(
            text("""
                SELECT MAX(created_at) as last_run
                FROM dedupe_suggestions
                WHERE account_id = :account_id
            """),
            {"account_id": account_id},
        )
        last_run = last_run_result.scalar()

        # Handle last_run which might be a string from SQLite
        last_run_formatted = None
        if last_run:
            if isinstance(last_run, str):
                # Already in ISO format from SQLite, just use it
                last_run_formatted = last_run
            else:
                # It's a datetime object, format it
                last_run_formatted = last_run.isoformat()

        return {
            "pending": pending,
            "accepted": accepted,
            "rejected": rejected,
            "total": total,
            "acceptance_rate": round(acceptance_rate, 1),
            "clusters_pending": clusters_pending,
            "pairs_merged": accepted,  # Each accepted suggestion is a pair merged
            "last_run_at": last_run_formatted,
        }

    except OperationalError as e:
        log.warning(_s(f"Dedupe metrics queries failed: {e}"))
        return default_metrics


# ============ Cluster Derivation and Merge Workbench ============


class UnionFind:
    """Union-Find data structure for cluster derivation"""

    def __init__(self):
        self.parent = {}
        self.rank = {}

    def find(self, x):
        if x not in self.parent:
            self.parent[x] = x
            self.rank[x] = 0
        if self.parent[x] != x:
            self.parent[x] = self.find(self.parent[x])
        return self.parent[x]

    def union(self, x, y):
        px, py = self.find(x), self.find(y)
        if px == py:
            return
        if self.rank[px] < self.rank[py]:
            px, py = py, px
        self.parent[py] = px
        if self.rank[px] == self.rank[py]:
            self.rank[px] += 1

    def get_components(self) -> List[Set[str]]:
        components_map = defaultdict(set)
        for node in self.parent:
            root = self.find(node)
            components_map[root].add(node)
        return list(components_map.values())


def _derive_anchor_from_details(pairs: List[Dict], details: List[Dict]) -> str:
    """Enhanced anchor detection from pair evidence"""
    from collections import defaultdict

    anchor_counts = defaultdict(int)
    anchor_scores = defaultdict(float)

    for detail in details:
        field = (detail.get("source_field") or detail.get("source_column", "")).lower()
        algo = (detail.get("algorithm", "")).lower()
        score = float(detail.get("score", 0))

        # Determine anchor type based on field, algorithm, and score
        anchor_type = None
        weight = 0

        if "email" in field:
            if algo == "exact" and score >= 0.95:
                anchor_type = "email"
                weight = 10  # Highest priority
            elif score >= 0.90:
                anchor_type = "email"
                weight = 8
        elif "phone" in field:
            if score >= 0.95:
                anchor_type = "phone_strong"
                weight = 9
            elif score >= 0.85:
                anchor_type = "phone"
                weight = 7
        elif "linkedin" in field:
            if score >= 0.95:
                anchor_type = "linkedin"
                weight = 8
        elif any(x in field for x in ["website", "domain"]):
            if score >= 0.90:
                anchor_type = "domain"
                weight = 6
        elif any(x in field for x in ["company", "name"]):
            if score >= 0.85:
                anchor_type = "name"
                weight = 4

        if anchor_type:
            anchor_counts[anchor_type] += 1
            anchor_scores[anchor_type] += weight * score

    # Check for manual cluster marker
    for pair in pairs:
        if "Manual" in pair.get("reasons", ""):
            return "manual"

    # Return highest weighted anchor
    if anchor_scores:
        best_anchor = max(anchor_scores.items(), key=lambda x: x[1])
        return best_anchor[0]

    # Fallback to count-based detection
    if anchor_counts:
        return max(anchor_counts.items(), key=lambda x: x[1])[0]

    return "fuzzy"  # Default fallback


def _compute_cluster_id(
    account_id: str, object_type: str, record_ids: List[str]
) -> str:
    """Generate deterministic cluster ID from sorted record IDs"""
    sorted_ids = sorted(set(record_ids))
    content = f"{account_id}|{object_type}|{'|'.join(sorted_ids)}"
    hash_val = hashlib.sha1(content.encode()).hexdigest()[:16]
    return f"clu_{hash_val}"


def _tier_from_score_simple(score: float) -> str:
    """Simple tier calculation without thresholds"""
    if score >= 0.95:
        return "CERTAIN"
    elif score >= 0.80:
        return "LIKELY"
    else:
        return "POSSIBLE"


def _field_support_for(
    record_id: str, field: str, pair_rows: List[Dict[str, Any]]
) -> float:
    """Calculate field support from evidence details for a specific record and field"""
    best = 0.0
    for row in pair_rows:
        details = _parse_details(row.get("field_score_details"))
        for d in details:
            src = d.get("source_column") or d.get("source_field") or ""
            ref = d.get("reference_column") or d.get("ref_field") or src
            if src == field or ref == field:
                score = _norm01(d.get("score", 0.0))
                if score > best:
                    best = score
    return best


# Object-specific SOQL fields to avoid errors
SOQL_FIELDS = {
    "Lead": [
        "Id",
        "Name",
        "Company",
        "Email",
        "Phone",
        "Owner.Name",
        "LastModifiedDate",
    ],
    "Contact": [
        "Id",
        "Name",
        "Email",
        "Phone",
        "Account.Name",
        "Owner.Name",
        "LastModifiedDate",
    ],
    "Account": ["Id", "Name", "Phone", "Website", "Owner.Name", "LastModifiedDate"],
}


async def _derive_clusters(
    db: AsyncSession,
    account_id: str,
    object_type: str,
    min_score: float = 0.0,
    search_query: str | None = None,
) -> List[Dict[str, Any]]:
    """Derive clusters from pending dedupe suggestions using Union-Find"""

    # Build SQL with optional search
    where_clauses = [
        "account_id = :account_id",
        "object_type = :object_type",
        "status = 'pending'",
        "score >= :min_score",
    ]

    params = {
        "account_id": account_id,
        "object_type": object_type,
        "min_score": min_score,
    }

    if search_query:
        where_clauses.append(
            "(reasons LIKE :search OR rec_id_1 LIKE :search OR rec_id_2 LIKE :search)"
        )
        params["search"] = f"%{search_query}%"

    # Fetch all pending pairs for this object type with performance guard
    sql = text(f"""
        SELECT rec_id_1, rec_id_2, score, tier, field_score_details, reasons
        FROM dedupe_suggestions
        WHERE {' AND '.join(where_clauses)}
        ORDER BY score DESC
        LIMIT 20000
    """)

    try:
        rows = (await db.execute(sql, params)).mappings().all()
    except OperationalError as e:
        if "no such table: dedupe_suggestions" in str(getattr(e, "orig", e)).lower():
            await ensure_dedupe_schema(db)
            return []
        raise

    if not rows:
        return []

    # Build union-find structure
    uf = UnionFind()
    pair_data = {}  # (rec1, rec2) -> {score, tier, details, reasons}

    for row in rows:
        rec1, rec2 = row["rec_id_1"], row["rec_id_2"]
        uf.union(rec1, rec2)
        # Store normalized pair (smaller id first)
        pair_key = (rec1, rec2) if rec1 < rec2 else (rec2, rec1)
        pair_data[pair_key] = {
            "score": float(row["score"]),
            "tier": row["tier"],
            "details": _parse_details(row["field_score_details"]),
            "reasons": row["reasons"],
        }

    # Extract components and build cluster info
    components = uf.get_components()
    clusters = []

    for component in components:
        record_ids = sorted(list(component))

        # Find all pairs within this cluster
        cluster_pairs = []
        for i, rec1 in enumerate(record_ids):
            for rec2 in record_ids[i + 1 :]:
                pair_key = (rec1, rec2) if rec1 < rec2 else (rec2, rec1)
                if pair_key in pair_data:
                    cluster_pairs.append(pair_data[pair_key])

        if not cluster_pairs:
            continue

        # Aggregate evidence from all pairs in cluster
        all_details = []
        max_score = 0.0
        top_tier = "POSSIBLE"

        for pair in cluster_pairs:
            all_details.extend(pair["details"])
            if pair["score"] > max_score:
                max_score = pair["score"]
                top_tier = pair["tier"]

        agg = _evidence_aggregate(all_details)

        # Enhanced anchor detection from pair evidence
        anchor = _derive_anchor_from_details(cluster_pairs, all_details)

        clusters.append(
            {
                "cluster_id": _compute_cluster_id(account_id, object_type, record_ids),
                "object_type": object_type,
                "record_ids": record_ids,
                "size": len(record_ids),
                "top_tier": top_tier,
                "max_score": max_score * 100,  # Convert to percentage for display
                "anchor": anchor,
                "evidence_count": agg["evidence_count"],
                "evidence_strength": agg["evidence_strength"],
            }
        )

    # Sort by max_score desc, then by size desc
    clusters.sort(key=lambda c: (-c["max_score"], -c["size"]))
    return clusters


@router.get("/clusters")
async def list_clusters(
    object_type: Literal["Lead", "Contact", "Account"],
    min_score: float = Query(0.0, ge=0.0, le=1.0),
    q: str | None = Query(None, description="Search query for reasons and IDs"),
    sort_by: str | None = Query(None, description="Sort by: score, tier, records"),
    sort_dir: str | None = Query(None, description="Sort direction: asc, desc"),
    page: int = Query(1, ge=1),
    page_size: int = Query(25, ge=1, le=100),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    """Get clusters of duplicate records derived from pending suggestions"""

    # Derive all clusters with search
    all_clusters = await _derive_clusters(db, account_id, object_type, min_score, q)

    # Sort clusters
    if sort_by:
        sort_key = None
        if sort_by.lower() == "score":
            sort_key = lambda c: c.get("average_score", 0)
        elif sort_by.lower() == "tier":
            tier_order = {"CERTAIN": 3, "LIKELY": 2, "POSSIBLE": 1}
            sort_key = lambda c: tier_order.get(c.get("tier", ""), 0)
        elif sort_by.lower() == "records":
            sort_key = lambda c: len(c.get("members", []))

        if sort_key:
            reverse = (sort_dir or "desc").lower() == "desc"
            all_clusters.sort(key=sort_key, reverse=reverse)

    # Paginate
    total = len(all_clusters)
    start_idx = (page - 1) * page_size
    end_idx = start_idx + page_size
    items = all_clusters[start_idx:end_idx]

    return {"items": items, "total": total, "page": page, "page_size": page_size}


@router.get("/preview")
async def preview_merge(
    cluster_id: str = Query(...),
    object_type: Literal["Lead", "Contact", "Account"] = Query(...),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """Preview merge details for a specific cluster"""

    # Derive clusters to find the requested one
    all_clusters = await _derive_clusters(db, account_id, object_type, 0.0)
    cluster = None
    for c in all_clusters:
        if c["cluster_id"] == cluster_id:
            cluster = c
            break

    if not cluster:
        raise HTTPException(status_code=404, detail="Cluster not found")

    record_ids = cluster["record_ids"]

    # Validate Salesforce IDs (15 or 18 alphanumeric chars only)
    safe_ids = [
        rid
        for rid in record_ids
        if isinstance(rid, str) and re.fullmatch(r"[A-Za-z0-9]{15,18}", rid)
    ]
    if not safe_ids:
        raise HTTPException(
            status_code=400, detail="No valid Salesforce IDs in cluster"
        )

    # SF gateway is injected via dependency - no need to create client

    # Fetch record details from Salesforce with validated IDs and object-specific fields
    fields = ", ".join(SOQL_FIELDS[object_type])
    id_list = "','".join(safe_ids)
    soql = f"SELECT {fields} FROM {object_type} WHERE Id IN ('{id_list}')"

    try:
        result = await sf.soql(soql)
        records = result.get("records", [])
    except Exception as e:
        log.error(f"Failed to fetch records from Salesforce: {e}")
        records = []

    # Simple fitness scoring for primary recommendation
    candidates = []
    best_score = 0
    best_id = safe_ids[0] if safe_ids else None

    for rec_id in safe_ids:
        # Find record data
        rec_data = next((r for r in records if r["Id"] == rec_id), None)

        # Simple fitness score based on completeness
        fitness = 0.0
        if rec_data:
            if rec_data.get("Email"):
                fitness += 0.3
            if rec_data.get("Phone"):
                fitness += 0.2
            if rec_data.get("Company"):
                fitness += 0.2
            if rec_data.get("Name"):
                fitness += 0.2
            if rec_data.get("LastModifiedDate"):
                # Prefer recently modified
                fitness += 0.1

        # Get evidence for this record from pairs
        rec_evidence_count = 0
        rec_evidence_strength = 0.0

        # Count pairs involving this record
        sql = text("""
            SELECT score, field_score_details
            FROM dedupe_suggestions
            WHERE account_id = :account_id
            AND object_type = :object_type
            AND status = 'pending'
            AND (rec_id_1 = :rec_id OR rec_id_2 = :rec_id)
        """)

        pair_rows = (
            (
                await db.execute(
                    sql,
                    {
                        "account_id": account_id,
                        "object_type": object_type,
                        "rec_id": rec_id,
                    },
                )
            )
            .mappings()
            .all()
        )

        if pair_rows:
            all_details = []
            for row in pair_rows:
                details = _parse_details(row["field_score_details"])
                all_details.extend(details)
            agg = _evidence_aggregate(all_details)
            rec_evidence_count = agg["evidence_count"]
            rec_evidence_strength = agg["evidence_strength"]

        # Combine fitness with evidence
        combined_score = fitness * 0.5 + rec_evidence_strength * 0.5

        if combined_score > best_score:
            best_score = combined_score
            best_id = rec_id

        candidates.append(
            {
                "id": rec_id,
                "score": combined_score,
                "tier": _tier_from_score_simple(combined_score),
                "evidence_count": rec_evidence_count,
                "evidence_strength": rec_evidence_strength,
            }
        )

    # Sort candidates by score
    candidates.sort(key=lambda c: -c["score"])

    # Build conflicts (simplified for V1)
    conflicts = []
    field_values = defaultdict(dict)  # field -> {record_id -> value}

    # Get fields relevant to this object type
    conflict_fields = {
        "Lead": ["Email", "Phone", "Company", "Name"],
        "Contact": ["Email", "Phone", "Name"],
        "Account": ["Phone", "Website", "Name"],
    }

    for rec in records:
        rec_id = rec["Id"]
        for field in conflict_fields.get(object_type, []):
            if field in rec and rec[field]:
                field_values[field][rec_id] = rec[field]
            # Handle nested Account.Name for Contacts
            elif (
                field == "Account"
                and "Account" in rec
                and rec["Account"]
                and "Name" in rec["Account"]
            ):
                field_values["AccountName"][rec_id] = rec["Account"]["Name"]

    for field, values_map in field_values.items():
        if len(set(values_map.values())) > 1:  # Different values exist
            options = []

            # Get all pair rows to calculate field-specific support
            all_pair_rows = []
            for rec_id in values_map.keys():
                sql = text("""
                    SELECT score, field_score_details
                    FROM dedupe_suggestions
                    WHERE account_id = :account_id
                    AND object_type = :object_type
                    AND status = 'pending'
                    AND (rec_id_1 = :rec_id OR rec_id_2 = :rec_id)
                """)

                rows = (
                    (
                        await db.execute(
                            sql,
                            {
                                "account_id": account_id,
                                "object_type": object_type,
                                "rec_id": rec_id,
                            },
                        )
                    )
                    .mappings()
                    .all()
                )
                all_pair_rows.extend(rows)

            for rec_id, value in values_map.items():
                # Calculate field-specific support from evidence
                support = _field_support_for(rec_id, field, all_pair_rows)
                options.append(
                    {"record_id": rec_id, "value": value, "support": support}
                )

            # Sort by support to pick default
            options.sort(key=lambda o: -o["support"])

            # Build reasons from actual evidence
            field_details = []
            for row in all_pair_rows:
                details = _parse_details(row.get("field_score_details"))
                for d in details:
                    if (d.get("source_column") or d.get("source_field") or "") == field:
                        field_details.append(d)

            conflicts.append(
                {
                    "field": field,
                    "options": options,
                    "default_record_id": options[0]["record_id"] if options else None,
                    "reasons": _reasons_from_details(field_details)
                    if field_details
                    else json.dumps({f"{field}->{field}": 100.0}),
                }
            )

    # Risk flags (simple heuristics for V1)
    risk_flags = []

    # Check for cross-owner
    owners = set()
    for rec in records:
        if rec.get("Owner") and rec["Owner"].get("Name"):
            owners.add(rec["Owner"]["Name"])
    if len(owners) > 1:
        risk_flags.append("CROSS_OWNER")

    # Check for different domains in emails
    domains = set()
    for rec in records:
        if rec.get("Email"):
            domain = rec["Email"].split("@")[-1].lower()
            domains.add(domain)
    if len(domains) > 1:
        risk_flags.append("DIFFERENT_DOMAINS")

    return {
        "cluster_id": cluster_id,
        "object_type": object_type,
        "primary_id": best_id,  # Changed from recommended_primary_id to match schema
        "candidates": candidates,
        "conflicts": conflicts,
        "children_summary": {
            "Activities": 0,  # V1 placeholder
            "Opportunities": 0,
            "Cases": 0,
            "Attachments": 0,
        },
        "risk_flags": risk_flags,
        "ensemble_used": cluster.get("evidence_count", 0) > 1,
    }


@router.get(
    "/merge-plan/{cluster_id}",
    response_model=MergePlanResponse,
    response_model_exclude_none=True,
)
async def get_merge_plan(
    cluster_id: str,
    object_type: Literal["Lead", "Contact", "Account"] = Query(...),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """
    Get comprehensive merge plan with all fields and policy suggestions.
    This powers the enhanced merge UI with field-level control.
    """
    # Find cluster
    clusters = await _derive_clusters(db, account_id, object_type, 0.0)
    cluster = next((c for c in clusters if c["cluster_id"] == cluster_id), None)
    if not cluster:
        raise HTTPException(404, "Cluster not found")

    # SF gateway is injected via dependency - no need to create client
    svc = PolicyDrivenDedupeService(db, account_id)

    # Fetch full records for cluster members
    import logging

    logger = logging.getLogger(__name__)

    all_recs = await svc.fetch_records(sf, object_type, save_integration=None)
    logger.info(f"Fetched {len(all_recs)} total {object_type} records")

    # Fix: Normalize IDs for comparison (Salesforce IDs are case-insensitive)
    cluster_ids_lower = [id.lower() for id in cluster["record_ids"]]
    logger.info(f"Looking for cluster IDs: {cluster_ids_lower}")

    # Check what IDs we actually have
    if all_recs:
        sample_ids = [r["Id"] for r in all_recs[:5]]
        logger.info(f"Sample fetched IDs: {sample_ids}")

    # Build case-insensitive mapping
    rec_by_id = {}
    for r in all_recs:
        if r["Id"].lower() in cluster_ids_lower:
            # Store with original ID as key
            rec_by_id[r["Id"]] = r
            # Also store with lowercase ID for lookup
            rec_by_id[r["Id"].lower()] = r

    logger.info(
        f"Found {len(rec_by_id) // 2 if rec_by_id else 0} records matching cluster IDs"
    )

    # If records weren't found in bulk fetch, try fetching specific IDs
    if (
        len(rec_by_id) < len(cluster["record_ids"]) * 2
    ):  # Times 2 because we store both cases
        missing_ids = [
            id
            for id in cluster["record_ids"]
            if not any(r["Id"].lower() == id.lower() for r in rec_by_id.values())
        ]
        if missing_ids:
            # Fetch missing records directly
            try:
                for record_id in missing_ids:
                    # Try with original case first
                    try:
                        record = await sf.get_record(object_type, record_id)
                        if record:
                            rec_by_id[record["Id"]] = record
                    except:
                        # Try uppercase (Salesforce IDs are typically uppercase)
                        try:
                            record = await sf.get_record(object_type, record_id.upper())
                            if record:
                                rec_by_id[record["Id"]] = record
                        except:
                            pass
            except Exception as e:
                # Log but don't fail
                import logging

                logging.warning(f"Could not fetch some cluster records: {e}")

    # FIX 5: Get all available fields with proper locking info
    schema = await sf.describe_object(object_type)

    # System fields that are always locked
    SYSTEM_FIELDS = {
        "Id",
        "CreatedDate",
        "CreatedById",
        "LastModifiedDate",
        "LastModifiedById",
        "SystemModstamp",
        "IsDeleted",
        "LastActivityDate",
        "LastViewedDate",
        "LastReferencedDate",
        "PhotoUrl",
    }

    # Build field list with lock status
    field_info = {}
    for f in schema.get("fields", []):
        field_name = f["name"]

        # Determine if field should be locked
        is_locked = (
            field_name in SYSTEM_FIELDS
            or not f.get("updateable", False)  # Not updateable
            or f.get("calculated", False)  # Formula field
            or f.get("externalId", False)  # External ID
            or f.get("autoNumber", False)  # Auto-number field
        )

        # Include all fields (even locked ones for display)
        field_info[field_name] = {
            "locked": is_locked,
            "type": f.get("type"),
            "label": f.get("label", field_name),
        }

    fields = list(field_info.keys())

    # Helper to normalize field values (reuse v3 normalizers)
    def normalize_value(field_name: str, value: Any) -> str:
        if not value:
            return ""
        field_lower = field_name.lower()
        if field_lower == "website":
            return svc.norm_domain(value)
        elif field_lower in ("phone", "mobilephone"):
            return svc.norm_phone(value) or ""
        elif field_lower == "email":
            return svc.norm_email(value) or ""
        else:
            # Generic normalization: lowercase, strip, handle None
            return str(value).strip().lower()

    # Helper to determine field group
    def get_field_group(obj_type: str, field_name: str) -> str:
        # System fields
        if field_name in (
            "OwnerId",
            "CreatedById",
            "CreatedDate",
            "LastModifiedById",
            "LastModifiedDate",
            "SystemModstamp",
        ):
            return "System"
        # Identity fields
        elif field_name in (
            "Name",
            "FirstName",
            "LastName",
            "Email",
            "Website",
            "LinkedIn__c",
        ):
            return "Identity"
        # Contact fields
        elif field_name in ("Phone", "MobilePhone", "Fax", "OtherPhone"):
            return "Contact"
        # Location fields
        elif any(
            x in field_name
            for x in [
                "Address",
                "Street",
                "City",
                "State",
                "Country",
                "PostalCode",
                "Zip",
            ]
        ):
            return "Location"
        # Company fields
        elif field_name in (
            "Company",
            "Title",
            "Department",
            "Industry",
            "NumberOfEmployees",
            "AnnualRevenue",
        ):
            return "Company"
        # Custom fields
        elif field_name.endswith("__c"):
            return "Custom"
        else:
            return "Other"

    # FIX 1: Get pair rows where BOTH IDs are in cluster (using expanding bind)
    from sqlalchemy.sql import bindparam

    ids = cluster["record_ids"]
    stmt = text("""
        SELECT rec_id_1, rec_id_2, field_score_details, score
        FROM dedupe_suggestions
        WHERE account_id = :account_id
          AND object_type = :object_type
          AND status = 'pending'
          AND rec_id_1 IN :ids
          AND rec_id_2 IN :ids
    """).bindparams(bindparam("ids", expanding=True))

    pair_rows = (
        (
            await db.execute(
                stmt, {"account_id": account_id, "object_type": object_type, "ids": ids}
            )
        )
        .mappings()
        .all()
    )

    # Build field matrix
    field_matrix = []
    for field_name in fields:
        # FIX 2: Two-pass approach for evidence calculation
        # 1) Gather normalized values for all records first
        normalized_values = {}
        for rid in cluster["record_ids"]:
            # Try both lowercase and uppercase when looking up record
            record = rec_by_id.get(rid) or rec_by_id.get(rid.upper()) or {}
            normalized_values[rid] = normalize_value(field_name, record.get(field_name))

        # 2) Precompute counts for identical values
        from collections import Counter

        counts = Counter(v for v in normalized_values.values() if v)

        # 3) Build options with evidence
        options = []
        for record_id in cluster["record_ids"]:
            # Try lowercase first since cluster IDs are lowercase
            record = rec_by_id.get(record_id) or rec_by_id.get(record_id.upper()) or {}
            raw = record.get(field_name)
            norm = normalized_values[record_id]
            evidence = 0.0

            # field_score_details signal from pair_rows
            for p in pair_rows:
                if record_id not in (p["rec_id_1"], p["rec_id_2"]):
                    continue
                for d in _parse_details(p.get("field_score_details")):
                    s = (d.get("source_column") or d.get("source_field") or "").lower()
                    r = (d.get("reference_column") or d.get("ref_field") or s).lower()
                    if field_name.lower() in (s, r):
                        evidence = max(evidence, _norm01(d.get("score", 0.0)))

            # fallback: identical normalized values across records
            if evidence == 0.0 and norm and counts[norm] > 1:
                evidence = 0.5

            flags = []
            if field_name.lower() == "email" and norm:
                flags.append("anchor_contrib")
            if (
                field_name.lower() == "website"
                and norm
                and not any(x in norm for x in ("gmail", "yahoo", "outlook"))
            ):
                flags.append("domain_verified")

            options.append(
                {
                    "record_id": record_id,
                    "value": raw,
                    "normalized": norm,
                    "evidence": evidence,
                    "flags": flags,
                }
            )

        # Detect conflicts (different normalized values)
        unique_normalized = set(nv for nv in normalized_values.values() if nv)
        has_conflict = len(unique_normalized) > 1

        # Suggest best value (highest evidence, prefer non-empty)
        suggested_id = cluster["record_ids"][0]  # default to first
        if options:
            non_empty = [o for o in options if o["value"]]
            if non_empty:
                suggested_id = max(non_empty, key=lambda o: o["evidence"])["record_id"]

        # FIX 5: Use proper lock status from field info
        is_locked = field_info.get(field_name, {}).get("locked", False)

        field_matrix.append(
            {
                "api": field_name,
                "label": field_info.get(field_name, {}).get("label", field_name),
                "group": get_field_group(object_type, field_name),
                "locked": is_locked,
                "hasConflict": has_conflict,
                "suggested_record_id": suggested_id,
                "confidence": max((o["evidence"] for o in options), default=0.0),
                "options": options,
            }
        )

    # FIX 4: Build candidate summaries with real scores from cluster
    candidates = []

    # Extract individual record scores from cluster pair data
    record_scores = {}
    for pair_row in pair_rows:
        rec1, rec2 = pair_row["rec_id_1"], pair_row["rec_id_2"]
        score = _norm01(pair_row.get("score", 0.5))
        # Track best score for each record
        for rec_id in [rec1, rec2]:
            if rec_id in cluster["record_ids"]:
                if rec_id not in record_scores or score > record_scores[rec_id]:
                    record_scores[rec_id] = score

    for record_id in cluster["record_ids"]:
        # Try lowercase first since cluster IDs are lowercase
        record = rec_by_id.get(record_id) or rec_by_id.get(record_id.upper()) or {}

        # Use actual score from cluster or fall back to average
        score = record_scores.get(record_id, cluster.get("avg_score", 0.75))

        # Calculate tier based on score thresholds
        if score >= 0.95:
            tier = "CERTAIN"
        elif score >= 0.85:
            tier = "HIGH"
        elif score >= 0.75:
            tier = "MEDIUM"
        else:
            tier = "POSSIBLE"

        candidates.append(
            {
                "id": record_id,
                "label": record.get("Name") or record.get("Company") or record_id[:8],
                "tier": tier,
                "score": score,
            }
        )

    # FIX 3 (CORRECTED): Load real policy version and derive anchor from pair data
    # Get actual policy
    from sqlalchemy import select

    policy_result = await db.execute(
        select(DedupePolicy).where(
            DedupePolicy.account_id == account_id,
            DedupePolicy.object_type == object_type,
            DedupePolicy.is_active == True,
        )
    )
    policy = policy_result.scalar_one_or_none()

    # FIX 3: Robust anchor derivation from pair details
    def derive_anchor_from_pairs(pairs):
        """Derive anchor type from pair data details."""
        for p in pairs:
            for d in _parse_details(p.get("field_score_details")):
                alg = (d.get("algorithm") or "").lower()
                s = (d.get("source_column") or d.get("source_field") or "").lower()
                r = (d.get("reference_column") or d.get("ref_field") or s).lower()
                pr = f"{s}->{r}"
                if "email" in pr and "exact" in alg:
                    return "email"
                if "phone" in pr and "exact" in alg:
                    return "phone"
                if any(k in pr for k in ("website", "domain")):
                    return "domain"
                if any(
                    k in pr
                    for k in ("linkedin", "external", "registration", "ein", "abn")
                ):
                    return "external_id"
        return "unknown"

    anchor_type = derive_anchor_from_pairs(pair_rows)

    # Determine primary_id with better logic
    # Use record with highest score if no explicit anchor
    primary_id = None
    if record_scores:
        primary_id = max(record_scores.items(), key=lambda x: x[1])[0]
    if not primary_id and cluster["record_ids"]:
        primary_id = cluster["record_ids"][0]

    # Improved guardrail computation
    def _iter_fsd_from_pairs_and_candidates(pair_rows, candidates, parse_details):
        yielded = False
        if pair_rows:
            for pair in pair_rows:
                fsd = parse_details(pair.get("field_score_details") or [])
                if fsd:
                    yielded = True
                    for d in fsd:
                        yield d
        if (not yielded) and candidates:
            for cand in candidates:
                for d in cand.get("field_score_details") or []:
                    yield d

    try:
        fsd_iter = _iter_fsd_from_pairs_and_candidates(
            pair_rows, candidates, _parse_details
        )
    except NameError:
        fsd_iter = _iter_fsd_from_pairs_and_candidates([], candidates, lambda x: x)

    has_true_anchor = False
    for d in fsd_iter:
        alg = (d.get("algorithm", "") or "").lower()
        sc = float(d.get("score") or 0.0)
        if sc > 1.0:
            sc /= 100.0
        if (("domain" in alg) or ("exact" in alg)) and sc >= 0.95:
            has_true_anchor = True
            break

    # Determine guardrail and reason
    cluster_guardrail = None
    cluster_guardrail_reason = None
    if not has_true_anchor:
        cluster_guardrail = "weak_anchor"
        cluster_guardrail_reason = (
            "No exact email/domain anchor found - manual review recommended"
        )

    # Transform field_matrix to match FieldValue schema
    fields = []
    for f in field_matrix:
        # Convert options array to values dict (record_id -> value)
        values_map = {}
        for opt in f.get("options") or []:
            values_map[opt["record_id"]] = opt.get("value")

        fields.append(
            {
                "field": f["api"],  # Schema expects 'field', not 'api'
                "label": f["label"],
                "group": f["group"],
                "hasConflict": f["hasConflict"],
                "values": values_map,  # Dict mapping instead of options array
                "locked": f["locked"],
            }
        )

    # Compute conflict count from transformed fields
    conflict_count = sum(1 for f in fields if f["hasConflict"])

    payload = {
        "cluster_id": cluster_id,
        "object_type": object_type,
        "primary_id": primary_id,  # Already using primary_id, not recommended_primary_id
        "policy_version": getattr(policy, "policy_version", None)
        or getattr(policy, "version", None)
        or "default",
        "anchor": anchor_type,
        "candidates": candidates,
        "fields": fields,  # Use transformed fields that match FieldValue schema
        "conflict_count": conflict_count,
        "risk_flags": [],  # Could add cross-owner, different-domain checks
        "hierarchy_context": {
            "relationship": "none"
        },  # TODO: wire Parent/Ultimate later
        "cluster_guardrail": cluster_guardrail,
        "cluster_guardrail_reason": cluster_guardrail_reason,
    }

    return payload


class MergeApplyRequest(BaseModel):
    cluster_id: str
    object_type: Literal["Lead", "Contact", "Account"]
    primary_id: str
    secondary_ids: List[str]
    field_overrides: Optional[Dict[str, Dict[str, str]]] = {}
    children_policy: Optional[Dict[str, bool]] = {"reparent": True}
    idempotency_key: Optional[str] = None


@router.post("/apply")
async def apply_merge(
    payload: MergeApplyRequest,
    request: Request = None,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    """Apply merge decision for a cluster"""

    # Create audit table if not exists
    await db.execute(
        text("""
        CREATE TABLE IF NOT EXISTS dedupe_merge_audit (
            id TEXT PRIMARY KEY,
            account_id TEXT NOT NULL,
            cluster_id TEXT NOT NULL,
            object_type TEXT NOT NULL,
            primary_id TEXT NOT NULL,
            secondary_ids TEXT NOT NULL,
            field_overrides TEXT,
            children_policy TEXT,
            idempotency_key TEXT,
            status TEXT DEFAULT 'pending',
            created_at TIMESTAMP NOT NULL,
            completed_at TIMESTAMP,
            error_message TEXT,
            UNIQUE(account_id, idempotency_key)
        )
    """)
    )
    await db.commit()

    # Check idempotency
    if payload.idempotency_key:
        existing = (
            (
                await db.execute(
                    text("""
            SELECT id, status FROM dedupe_merge_audit
            WHERE account_id = :account_id AND idempotency_key = :key
        """),
                    {
                        "account_id": account_id,
                        "idempotency_key": payload.idempotency_key,
                    },
                )
            )
            .mappings()
            .first()
        )

        if existing:
            return {
                "merge_job_id": existing["id"],
                "status": existing["status"],
                "message": "Idempotent request - returning existing job",
            }

    # Create EventEmitter and ChangeSet for this merge
    emitter = EventEmitter()
    cs_id = str(uuid.uuid4())
    cs = ChangeSet(
        id=cs_id,
        account_id=str(account_id),
        name="Dedupe Merge",
        type="dedupe_merge",
        source="ui",
        actor_type="user",
        actor_id=request.headers.get("X-User-Email", "system") if request else "system",
        status="open",
    )
    db.add(cs)
    await db.flush()

    # Create audit record
    job_id = f"job_{uuid.uuid4().hex[:12]}"
    now = datetime.utcnow().isoformat(sep=" ", timespec="seconds")

    await db.execute(
        text("""
        INSERT INTO dedupe_merge_audit
        (id, account_id, cluster_id, object_type, primary_id, secondary_ids,
         field_overrides, children_policy, idempotency_key, status, created_at)
        VALUES (:id, :account_id, :cluster_id, :object_type, :primary_id, :secondary_ids,
                :field_overrides, :children_policy, :idempotency_key, 'queued', :created_at)
    """),
        {
            "id": job_id,
            "account_id": account_id,
            "cluster_id": payload.cluster_id,
            "object_type": payload.object_type,
            "primary_id": payload.primary_id,
            "secondary_ids": json.dumps(payload.secondary_ids),
            "field_overrides": json.dumps(payload.field_overrides or {}),
            "children_policy": json.dumps(payload.children_policy or {}),
            "idempotency_key": payload.idempotency_key,
            "created_at": now,
        },
    )

    # Update dedupe_suggestions to remove cluster from queue
    all_ids = [payload.primary_id] + payload.secondary_ids

    # Mark suggestions as accepted/rejected
    for i, id1 in enumerate(all_ids):
        for id2 in all_ids[i + 1 :]:
            # Normalize pair order
            rec1, rec2 = (id1, id2) if id1 < id2 else (id2, id1)

            # If both are in the merge, mark as accepted
            # If one is primary and other is secondary, also accepted
            # This removes the cluster from pending queue
            actor = "system"
            if request and hasattr(request, "headers"):
                actor = request.headers.get("X-User-Email", "system")

            await db.execute(
                text("""
                UPDATE dedupe_suggestions
                SET status = 'accepted', decided_at = :now, decided_by = :actor
                WHERE account_id = :account_id
                AND object_type = :object_type
                AND (
                    (rec_id_1 = :rec1 AND rec_id_2 = :rec2)  -- normalized order
                    OR (rec_id_1 = :rec2 AND rec_id_2 = :rec1)  -- legacy reversed safety
                )
                AND status = 'pending'
            """),
                {
                    "account_id": account_id,
                    "object_type": payload.object_type,
                    "rec1": rec1,
                    "rec2": rec2,
                    "now": now,
                    "actor": actor,
                },
            )

    # Emit merge events
    events = []
    # Create merge event with compressed preimage for potential unmerge
    preimage_bundle = {
        "primary_id": payload.primary_id,
        "secondary_ids": payload.secondary_ids,
        "field_overrides": payload.field_overrides or {},
        "cluster_id": payload.cluster_id,
    }

    # Main merge event
    events.append(
        {
            "changeset_id": cs_id,
            "object_type": payload.object_type,
            "object_id": payload.primary_id,
            "op": "merge",
            "field": None,
            "before_value": json.dumps(preimage_bundle),  # Store preimage for unmerge
            "after_value": json.dumps({"merged_ids": payload.secondary_ids}),
            "confidence": 0.95,  # High confidence for user-approved merges
            "reason_text": f"Merged {len(payload.secondary_ids)} records into primary",
            "actor_type": "user",
            "actor_id": request.headers.get("X-User-Email", "system")
            if request
            else "system",
        }
    )

    # Emit field override events if any
    for field_name, field_value in (payload.field_overrides or {}).items():
        events.append(
            {
                "changeset_id": cs_id,
                "object_type": payload.object_type,
                "object_id": payload.primary_id,
                "op": "update_field",
                "field": field_name,
                "before_value": None,  # Would need to fetch from SF for true before value
                "after_value": field_value,
                "confidence": 1.0,
                "reason_text": "Field override during merge",
                "actor_type": "user",
                "actor_id": request.headers.get("X-User-Email", "system")
                if request
                else "system",
            }
        )

    if events:
        await emitter.emit_batch(db, str(account_id), events)

    # Finalize changeset
    cs.status = "finalized"
    cs.attempted_count = len(events)
    cs.applied_count = len(events)
    cs.failed_count = 0
    cs.finalized_at = _dt.datetime.utcnow()

    await db.commit()

    # V1: Log the merge plan but don't execute actual Salesforce merge
    log.info(f"Merge job {job_id} created for cluster {payload.cluster_id}")
    log.info(f"Primary: {payload.primary_id}, Secondaries: {payload.secondary_ids}")
    log.info(f"Field overrides: {payload.field_overrides}")

    # TODO: In V2, add actual Salesforce merge API call here

    return {
        "merge_job_id": job_id,
        "status": "queued",
        "message": "Merge job created successfully",
    }


@router.post("/clusters/manual")
async def create_manual_cluster(
    request: Request,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    """Create a manual cluster by adding pairs between specified records"""
    body = await request.json()

    object_type = body.get("object_type", "Lead")
    record_ids = body.get("record_ids", [])
    reason = body.get("reason", "Manual duplicate identification")
    confidence = body.get("confidence", 1.0)

    if len(record_ids) < 2:
        raise HTTPException(400, "At least 2 record IDs required")

    # Ensure dedupe schema exists
    await ensure_dedupe_schema(db)

    # Create pairs between all records (fully connected)
    pairs_created = 0
    now = datetime.now().isoformat(sep=" ", timespec="seconds")

    for i, rec1 in enumerate(record_ids):
        for rec2 in record_ids[i + 1 :]:
            # Insert manual pair as pending suggestion
            await db.execute(
                text("""
                INSERT INTO dedupe_suggestions
                (account_id, object_type, rec_id_1, rec_id_2, score, tier, status,
                 reasons, created_at)
                VALUES
                (:account_id, :object_type, :rec_id_1, :rec_id_2, :score, :tier,
                 'pending', :reasons, :created_at)
                ON CONFLICT (account_id, object_type, rec_id_1, rec_id_2)
                DO UPDATE SET
                    score = excluded.score,
                    tier = excluded.tier,
                    reasons = excluded.reasons
            """),
                {
                    "account_id": account_id,
                    "object_type": object_type,
                    "rec_id_1": rec1 if rec1 < rec2 else rec2,
                    "rec_id_2": rec2 if rec1 < rec2 else rec1,
                    "score": confidence,
                    "tier": "CERTAIN"
                    if confidence >= 0.95
                    else "LIKELY"
                    if confidence >= 0.8
                    else "POSSIBLE",
                    "reasons": reason,
                    "created_at": now,
                },
            )
            pairs_created += 1

    await db.commit()

    # Compute cluster ID for this group
    cluster_id = _compute_cluster_id(account_id, object_type, sorted(record_ids))

    return {
        "cluster_id": cluster_id,
        "object_type": object_type,
        "record_ids": record_ids,
        "pairs_created": pairs_created,
        "status": "success",
    }


@router.post("/clusters/{cluster_id}/skip")
async def skip_cluster(
    cluster_id: str,
    object_type: Literal["Lead", "Contact", "Account"] = Query(...),
    request: Request = None,
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
):
    """Skip a cluster by marking all its pairs as rejected"""

    # Find cluster record_ids
    clusters = await _derive_clusters(db, account_id, object_type, 0.0)
    cluster = next((x for x in clusters if x["cluster_id"] == cluster_id), None)
    if not cluster:
        raise HTTPException(status_code=404, detail="Cluster not found")

    ids = cluster["record_ids"]
    now = datetime.utcnow().isoformat(sep=" ", timespec="seconds")
    actor = "system"
    if request and hasattr(request, "headers"):
        actor = request.headers.get("X-User-Email", "system")

    # Mark all pending pairs in that cluster as rejected
    count = 0
    for i, a in enumerate(ids):
        for b in ids[i + 1 :]:
            rec1, rec2 = (a, b) if a < b else (b, a)
            result = await db.execute(
                text("""
                UPDATE dedupe_suggestions
                SET status = 'rejected', decided_at = :now, decided_by = :actor
                WHERE account_id = :account_id
                AND object_type = :object_type
                AND (
                    (rec_id_1 = :rec1 AND rec_id_2 = :rec2)
                    OR (rec_id_1 = :rec2 AND rec_id_2 = :rec1)
                )
                AND status = 'pending'
            """),
                {
                    "account_id": account_id,
                    "object_type": object_type,
                    "rec1": rec1,
                    "rec2": rec2,
                    "now": now,
                    "actor": actor,
                },
            )
            if result.rowcount > 0:
                count += 1

    await db.commit()

    return {"cluster_id": cluster_id, "status": "skipped", "pairs_rejected": count}


@router.get("/cross-check/test")
async def test_cross_check():
    """Simple test endpoint to verify router is working"""
    return {"status": "ok", "message": "Cross-check endpoints are registered"}


@router.get("/cross-check/lead-contact")
async def check_lead_contact_overlaps(
    min_score: float = Query(0.75, ge=0.0, le=1.0),
    limit: int = Query(100, le=500),
    initial_batch: int = Query(
        100, ge=50, le=500, description="Initial batch size for progressive loading"
    ),
    load_more: bool = Query(
        False, description="Load additional results beyond initial batch"
    ),
    db: AsyncSession = Depends(get_session),
    account_id: str = Depends(get_current_account),
    sf: SalesforceGateway = Depends(get_salesforce_gateway),
):
    """
    Detect potential Lead-Contact duplicates using existing match infrastructure.
    Read-only analysis - no modifications.
    """
    import pandas as pd
    from ...intelligence.similarity import fuzz

    start_time = time.perf_counter()

    log.info(
        f"Cross-object detection called - account_id: {account_id}, min_score: {min_score}"
    )

    # SF gateway is injected via dependency - no need to create client

    # Determine batch size based on progressive loading
    batch_size = 500 if load_more else initial_batch

    # Fetch sample data with error handling
    try:
        leads_result = await sf.soql(
            f"SELECT Id, FirstName, LastName, Email, Company, Phone "
            f"FROM Lead WHERE IsConverted = false LIMIT {batch_size}"
        )

        contacts_result = await sf.soql(
            f"SELECT Id, FirstName, LastName, Email, Phone, Account.Name "
            f"FROM Contact WHERE IsDeleted = false LIMIT {batch_size}"
        )

    except Exception as e:
        log.error(f"SOQL query failed: {e}")
        return {
            "overlaps": [],
            "stats": {
                "leads_checked": 0,
                "contacts_checked": 0,
                "overlaps_found": 0,
                "error": str(e),
            },
        }

    leads = leads_result.get("records", [])
    contacts = contacts_result.get("records", [])

    if not leads or not contacts:
        return {
            "overlaps": [],
            "stats": {
                "leads_checked": len(leads),
                "contacts_checked": len(contacts),
                "overlaps_found": 0,
                "message": "No leads or contacts found",
            },
        }

    # Convert to DataFrames for easier processing
    lead_df = pd.DataFrame(leads)
    contact_df = pd.DataFrame(contacts)

    # Handle nested Account.Name for contacts
    if "Account" in contact_df.columns:
        contact_df["Company"] = contact_df["Account"].apply(
            lambda x: x.get("Name") if isinstance(x, dict) else None
        )

    # Clean up DataFrames - remove Salesforce metadata
    for col in ["attributes", "Account"]:
        if col in lead_df.columns:
            lead_df = lead_df.drop(columns=[col])
        if col in contact_df.columns:
            contact_df = contact_df.drop(columns=[col])

    # Find overlaps using fuzzy matching
    overlaps = []

    # Pre-index by email for O(n) performance
    email_to_contacts = {}
    for _, contact in contact_df.iterrows():
        email = str(contact.get("Email", "")).lower().strip()
        if email and email != "nan":
            if email not in email_to_contacts:
                email_to_contacts[email] = []
            email_to_contacts[email].append(contact)

    # Check each lead
    for _, lead in lead_df.iterrows():
        lead_email = str(lead.get("Email", "")).lower().strip()
        lead_name = f"{lead.get('FirstName', '')} {lead.get('LastName', '')}".strip()

        # Check for email matches first (fastest)
        if lead_email and lead_email != "nan" and lead_email in email_to_contacts:
            for contact in email_to_contacts[lead_email]:
                contact_name = f"{contact.get('FirstName', '')} {contact.get('LastName', '')}".strip()

                # Calculate name similarity
                name_score = (
                    fuzz.ratio(lead_name, contact_name) / 100.0
                    if lead_name and contact_name
                    else 0
                )

                # Email exact match + name similarity
                final_score = 0.7 + (
                    0.3 * name_score
                )  # 70% weight on email, 30% on name

                if final_score >= min_score:
                    evidence = ["EMAIL_EXACT"]
                    if name_score > 0.8:
                        evidence.append("NAME_MATCH")
                    elif name_score > 0.6:
                        evidence.append("NAME_SIMILAR")

                    overlaps.append(
                        {
                            "lead_id": lead.get("Id"),
                            "lead_name": lead_name,
                            "lead_email": lead.get("Email"),
                            "lead_company": lead.get("Company"),
                            "contact_id": contact.get("Id"),
                            "contact_name": contact_name,
                            "contact_email": contact.get("Email"),
                            "account_name": contact.get("Company"),
                            "score": round(final_score, 2),
                            "match_type": "High"
                            if final_score >= 0.90
                            else "Medium"
                            if final_score >= 0.80
                            else "Low",
                            "evidence_flags": ",".join(evidence),
                        }
                    )

        # Also check name similarity for leads without email
        elif lead_name and not lead_email:
            for _, contact in contact_df.iterrows():
                contact_name = f"{contact.get('FirstName', '')} {contact.get('LastName', '')}".strip()

                if contact_name:
                    name_score = fuzz.ratio(lead_name, contact_name) / 100.0

                    if name_score >= min_score:
                        evidence = [
                            "NAME_MATCH" if name_score > 0.9 else "NAME_SIMILAR"
                        ]

                        # Check company match
                        lead_company = str(lead.get("Company", "")).lower().strip()
                        contact_company = (
                            str(contact.get("Company", "")).lower().strip()
                        )

                        if (
                            lead_company
                            and contact_company
                            and lead_company != "nan"
                            and contact_company != "nan"
                        ):
                            company_score = (
                                fuzz.ratio(lead_company, contact_company) / 100.0
                            )
                            if company_score > 0.7:
                                evidence.append("COMPANY_SIMILAR")
                                name_score = (name_score + company_score) / 2

                        if name_score >= min_score:
                            overlaps.append(
                                {
                                    "lead_id": lead.get("Id"),
                                    "lead_name": lead_name,
                                    "lead_email": lead.get("Email"),
                                    "lead_company": lead.get("Company"),
                                    "contact_id": contact.get("Id"),
                                    "contact_name": contact_name,
                                    "contact_email": contact.get("Email"),
                                    "account_name": contact.get("Company"),
                                    "score": round(name_score, 2),
                                    "match_type": "High"
                                    if name_score >= 0.90
                                    else "Medium"
                                    if name_score >= 0.80
                                    else "Low",
                                    "evidence_flags": ",".join(evidence),
                                }
                            )

    # Sort by score descending
    overlaps.sort(key=lambda x: x["score"], reverse=True)

    execution_time_ms = int((time.perf_counter() - start_time) * 1000)

    return {
        "overlaps": overlaps[:limit],
        "stats": {
            "leads_checked": len(leads),
            "contacts_checked": len(contacts),
            "overlaps_found": len(overlaps),
            "high_confidence": sum(1 for o in overlaps if o["score"] >= 0.90),
            "medium_confidence": sum(1 for o in overlaps if 0.80 <= o["score"] < 0.90),
            "low_confidence": sum(1 for o in overlaps if o["score"] < 0.80),
            "execution_time_ms": execution_time_ms,
            "batch_size": batch_size,
            "can_load_more": not load_more and batch_size < 500,
            "from_cache": False,
        },
    }
