"""
Enterprise dedupe planning helpers for large CRM jobs.

This module adds a blocking-first offline planning workflow that can operate on
multiple CRM exports (for example Contacts + Leads), emit review-ready merge
artifacts, and surface the engine's multi-algorithm matching profile in the
resulting job manifest.
"""

from __future__ import annotations

import asyncio
import csv
import json
import logging
import re
from collections import defaultdict
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple

LOGGER = logging.getLogger(__name__)

COMMON_CONSUMER_DOMAINS = {
    "gmail.com",
    "googlemail.com",
    "hotmail.com",
    "icloud.com",
    "outlook.com",
    "yahoo.com",
    "aol.com",
    "msn.com",
    "live.com",
    "proton.me",
    "protonmail.com",
}

CANONICAL_KEY_COLUMNS = {
    "name": "__match_name",
    "company": "__match_company",
    "email": "__match_email",
    "phone": "__match_phone",
    "domain": "__match_domain",
}

DEFAULT_KEY_COLUMNS = ["name", "email", "phone", "company", "domain"]
DEFAULT_BLOCKING_KEYS = ["email", "phone", "domain_name", "company_name"]
ALLOWED_BLOCKING_KEYS = set(DEFAULT_BLOCKING_KEYS)
DEFAULT_PROFILE = "default"
PROFILE_DEFAULTS = {
    "default": {
        "key_columns": DEFAULT_KEY_COLUMNS,
        "blocking_keys": DEFAULT_BLOCKING_KEYS,
        "threshold": 85,
        "notes": [
            "Balanced CRM dedupe profile using exact anchors first, with domain and company-name blocking enabled.",
            "Good for broad CRM cleanup, but domain-name blocking can be too permissive for B2B person resolution.",
        ],
    },
    "b2b_person": {
        "key_columns": DEFAULT_KEY_COLUMNS,
        "blocking_keys": ["email", "phone", "company_name"],
        "threshold": 90,
        "notes": [
            "B2B person dedupe profile favors exact person anchors first, with company-name as a secondary exact block.",
            "Domain stays in scoring but is removed from default blocking to avoid coworker chaining for same-name contacts.",
        ],
    },
}
ALLOWED_PROFILES = set(PROFILE_DEFAULTS)
FULL_SCAN_FALLBACK_MAX_ROWS = 250
NOISY_COMPANY_NAMES = {
    "self employed",
    "freelance",
    "freelancer",
    "n a",
    "na",
    "none",
    "unknown",
    "individual",
    "independent",
}

ENGINE_CAPABILITIES = {
    "multi_algorithm": True,
    "algorithm_families": [
        "Exact",
        "JaroWinkler",
        "Jaro",
        "TokenSetRatio",
        "TokenSortRatio",
        "WRatio",
        "PartialRatio",
        "Domain",
    ],
    "field_ensembles": {
        "email": ["Exact", "JaroWinkler", "WRatio"],
        "phone": ["Exact", "JaroWinkler"],
        "domain": ["TokenSetRatio", "WRatio", "JaroWinkler"],
        "name_company": ["WRatio", "TokenSetRatio", "PartialRatio"],
    },
    "notes": [
        "Blocking-first candidate generation keeps large CRM dedupe jobs tractable.",
        "Cross-object clusters are flagged for Lead-to-Contact resolution instead of blind merge.",
    ],
}


@dataclass(frozen=True)
class SourceSpec:
    object_type: str
    csv_path: Path


@dataclass
class Cluster:
    members: List[Dict[str, Any]]
    golden_index: int
    merge_confidence: float
    anchors: List[str] = field(default_factory=list)


class _UnionFind:
    def __init__(self, values: Iterable[Any]):
        self.parents = {value: value for value in values}

    def find(self, value: Any) -> Any:
        while self.parents[value] != value:
            self.parents[value] = self.parents[self.parents[value]]
            value = self.parents[value]
        return value

    def union(self, left: Any, right: Any) -> None:
        root_left = self.find(left)
        root_right = self.find(right)
        if root_left != root_right:
            self.parents[root_right] = root_left


def _normalize_text(value: Any) -> str:
    return re.sub(r"[^a-z0-9]+", " ", str(value or "").lower()).strip()


def _normalize_email(value: Any) -> str:
    return str(value or "").strip().lower()


def _normalize_phone(value: Any) -> str:
    digits = re.sub(r"\D+", "", str(value or ""))
    if len(digits) == 11 and digits.startswith("1"):
        return digits[1:]
    return digits


def _normalize_domain(value: Any) -> str:
    text = str(value or "").strip().lower()
    if not text:
        return ""
    if "@" in text and "/" not in text:
        text = text.split("@", 1)[1]
    text = re.sub(r"^https?://", "", text)
    text = text.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0].strip()
    if text.startswith("www."):
        text = text[4:]
    return text


def _load_csv(csv_path: Path, *, limit: int | None = None) -> Tuple[List[Dict[str, str]], List[str]]:
    with csv_path.open(newline="", encoding="utf-8-sig") as handle:
        reader = csv.DictReader(handle)
        headers = list(reader.fieldnames or [])
        rows: List[Dict[str, str]] = []
        for row in reader:
            rows.append(row)
            if limit is not None and len(rows) >= limit:
                break
    if not rows:
        return [], headers
    return rows, headers


def _header_index(headers: Iterable[str]) -> Dict[str, str]:
    return {str(header).strip().lower(): str(header) for header in headers}


def _resolve_header(headers: Iterable[str], candidates: Sequence[str]) -> Optional[str]:
    header_map = _header_index(headers)
    for candidate in candidates:
        actual = header_map.get(candidate.lower())
        if actual:
            return actual
    return None


def _first_non_blank(row: Dict[str, Any], columns: Sequence[Optional[str]]) -> str:
    for column in columns:
        if not column:
            continue
        value = str(row.get(column, "") or "").strip()
        if value:
            return value
    return ""


def _compose_name(
    row: Dict[str, Any],
    *,
    name_column: Optional[str],
    first_name_column: Optional[str],
    last_name_column: Optional[str],
) -> str:
    direct = _first_non_blank(row, [name_column])
    if direct:
        return direct
    first_name = _first_non_blank(row, [first_name_column])
    last_name = _first_non_blank(row, [last_name_column])
    return " ".join(part for part in [first_name, last_name] if part).strip()


def _detect_columns(headers: Sequence[str]) -> Dict[str, Optional[str]]:
    return {
        "id": _resolve_header(headers, ["Id", "RecordId", "record_id", "ID", "sfid"]),
        "name": _resolve_header(headers, ["Name", "FullName", "Full Name", "Contact Name", "Lead Name"]),
        "first_name": _resolve_header(headers, ["FirstName", "First Name", "first_name", "given_name"]),
        "last_name": _resolve_header(headers, ["LastName", "Last Name", "last_name", "surname", "family_name"]),
        "company": _resolve_header(
            headers,
            ["Company", "Company Name", "Account Name", "AccountName", "Account", "Organization", "Org"],
        ),
        "email": _resolve_header(
            headers,
            ["Email", "Email Address", "EmailAddress", "work_email", "business_email", "email"],
        ),
        "phone": _resolve_header(
            headers,
            [
                "Phone",
                "Phone Number",
                "phone",
                "MobilePhone",
                "Mobile Phone",
                "Business Phone",
                "Work Phone",
            ],
        ),
        "domain": _resolve_header(
            headers,
            ["Domain", "Website", "Website Domain", "Company Domain", "web", "url", "URL"],
        ),
    }


def _prepare_source_rows(
    rows: Sequence[Dict[str, str]],
    headers: Sequence[str],
    source: SourceSpec,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    detected = _detect_columns(headers)
    prepared: List[Dict[str, Any]] = []
    for index, row in enumerate(rows, start=1):
        display_name = _compose_name(
            row,
            name_column=detected["name"],
            first_name_column=detected["first_name"],
            last_name_column=detected["last_name"],
        )
        display_company = _first_non_blank(row, [detected["company"]])
        display_email = _first_non_blank(row, [detected["email"]])
        display_phone = _first_non_blank(row, [detected["phone"]])
        raw_domain = _first_non_blank(row, [detected["domain"]])
        display_domain = _normalize_domain(raw_domain or display_email)
        record_id = _first_non_blank(row, [detected["id"]]) or f"{source.object_type}:{source.csv_path.stem}:{index}"

        prepared_row = dict(row)
        prepared_row.update(
            {
                "__object_type": source.object_type,
                "__source_file": source.csv_path.name,
                "__record_id": record_id,
                "__display_name": display_name,
                "__display_company": display_company,
                "__display_email": display_email,
                "__display_phone": display_phone,
                "__display_domain": display_domain,
                "__match_name": _normalize_text(display_name),
                "__match_company": _normalize_text(display_company),
                "__match_email": _normalize_email(display_email),
                "__match_phone": _normalize_phone(display_phone),
                "__match_domain": display_domain,
            }
        )
        prepared.append(prepared_row)

    source_meta = {
        "object_type": source.object_type,
        "path": str(source.csv_path),
        "rows": len(prepared),
        "detected_columns": detected,
    }
    return prepared, source_meta


def _resolve_key_columns(requested: Sequence[str], available_columns: Iterable[str]) -> List[str]:
    available_map = _header_index(available_columns)
    resolved: List[str] = []
    missing: List[str] = []
    for token in requested:
        key = str(token or "").strip()
        if not key:
            continue
        canonical = CANONICAL_KEY_COLUMNS.get(key.lower())
        if canonical:
            resolved.append(canonical)
            continue
        actual = available_map.get(key.lower())
        if actual:
            resolved.append(actual)
            continue
        missing.append(key)
    if missing:
        raise ValueError(f"Key columns not found: {', '.join(missing)}")
    if not resolved:
        raise ValueError("At least one match key column is required.")
    return resolved


def _resolve_profile_settings(
    *,
    profile: str,
    key_columns: Sequence[str] | None,
    blocking_keys: Sequence[str] | None,
    threshold: int | None,
) -> Dict[str, Any]:
    profile_name = str(profile or DEFAULT_PROFILE).strip().lower() or DEFAULT_PROFILE
    if profile_name not in ALLOWED_PROFILES:
        raise ValueError(f"Unsupported dedupe profile: {profile_name}")

    defaults = PROFILE_DEFAULTS[profile_name]
    key_column_source = defaults["key_columns"] if key_columns is None else key_columns
    blocking_key_source = defaults["blocking_keys"] if blocking_keys is None else blocking_keys
    resolved_key_columns = [str(key).strip() for key in key_column_source if str(key).strip()]
    resolved_blocking_keys = [
        str(key).strip().lower() for key in blocking_key_source if str(key).strip()
    ]
    resolved_threshold = int(defaults["threshold"] if threshold is None else threshold)
    return {
        "profile": profile_name,
        "key_columns": resolved_key_columns,
        "blocking_keys": resolved_blocking_keys,
        "threshold": resolved_threshold,
        "notes": list(defaults.get("notes") or []),
        "defaults_applied": {
            "key_columns": key_columns is None,
            "blocking_keys": blocking_keys is None,
            "threshold": threshold is None,
        },
    }


def _candidate_anchor_keys(row: Dict[str, Any], blocking_keys: Sequence[str]) -> List[Tuple[str, str]]:
    anchors: List[Tuple[str, str]] = []
    name = row.get("__match_name", "") or ""
    company = row.get("__match_company", "") or ""
    email = row.get("__match_email", "") or ""
    phone = row.get("__match_phone", "") or ""
    domain = row.get("__match_domain", "") or ""

    if "email" in blocking_keys and email:
        anchors.append(("email", email))
    if "phone" in blocking_keys and len(phone) >= 7:
        anchors.append(("phone", phone))
    if "domain_name" in blocking_keys and domain and name and domain not in COMMON_CONSUMER_DOMAINS:
        anchors.append(("domain_name", f"{domain}|{name}"))
    if "company_name" in blocking_keys and company and name and company not in NOISY_COMPANY_NAMES:
        anchors.append(("company_name", f"{company}|{name}"))
    return anchors


def _build_candidate_components(
    rows: Sequence[Dict[str, Any]],
    blocking_keys: Sequence[str],
    block_size_cap: int,
) -> Tuple[List[List[int]], Dict[str, Any], Dict[int, set[str]]]:
    anchor_groups: Dict[Tuple[str, str], List[int]] = defaultdict(list)
    row_anchor_kinds: Dict[int, set[str]] = defaultdict(set)
    active_rows_by_kind: Dict[str, set[int]] = defaultdict(set)
    group_counts: Dict[str, int] = defaultdict(int)
    ignored_counts: Dict[str, int] = defaultdict(int)
    ignored_examples: List[Dict[str, Any]] = []

    for row in rows:
        row_index = int(row.get("__row_index", -1))
        if row_index < 0:
            continue
        seen_for_row: set[Tuple[str, str]] = set()
        for anchor in _candidate_anchor_keys(row, blocking_keys):
            if anchor in seen_for_row:
                continue
            seen_for_row.add(anchor)
            anchor_groups[anchor].append(row_index)

    union_find = _UnionFind(range(len(rows)))

    for (anchor_kind, anchor_value), member_ids in anchor_groups.items():
        unique_ids = sorted(set(member_ids))
        if len(unique_ids) < 2:
            continue
        if len(unique_ids) > block_size_cap:
            ignored_counts[anchor_kind] += 1
            if len(ignored_examples) < 10:
                ignored_examples.append(
                    {"anchor": anchor_kind, "size": len(unique_ids), "sample": str(anchor_value)[:80]}
                )
            continue

        group_counts[anchor_kind] += 1
        active_rows_by_kind[anchor_kind].update(unique_ids)
        for member_id in unique_ids:
            row_anchor_kinds[member_id].add(anchor_kind)
        for member_id in unique_ids[1:]:
            union_find.union(unique_ids[0], member_id)

    components_map: Dict[int, List[int]] = defaultdict(list)
    for member_id in sorted(row_anchor_kinds):
        components_map[union_find.find(member_id)].append(member_id)

    components = [sorted(member_ids) for member_ids in components_map.values() if len(member_ids) > 1]
    fallback_full_scan = False
    if not components and len(rows) <= FULL_SCAN_FALLBACK_MAX_ROWS:
        components = [list(range(len(rows)))] if len(rows) > 1 else []
        fallback_full_scan = bool(components)
        for idx in range(len(rows)):
            row_anchor_kinds.setdefault(idx, set())

    summary = {
        "blocking_keys": list(blocking_keys),
        "candidate_components": len(components),
        "candidate_rows": len({member_id for component in components for member_id in component}),
        "largest_component": max((len(component) for component in components), default=0),
        "fallback_full_scan": fallback_full_scan,
        "anchors": {
            key: {
                "groups": group_counts.get(key, 0),
                "rows": len(active_rows_by_kind.get(key, set())),
                "ignored_groups": ignored_counts.get(key, 0),
            }
            for key in blocking_keys
        },
        "ignored_examples": ignored_examples,
    }
    return components, summary, row_anchor_kinds


def _sequence_similarity(left: str, right: str) -> float:
    return SequenceMatcher(None, left, right).ratio() * 100.0


def _column_similarity(column: str, left: Any, right: Any) -> Optional[float]:
    left_text = str(left or "").strip()
    right_text = str(right or "").strip()
    if not left_text or not right_text:
        return None

    lowered = column.lower()
    if "email" in lowered:
        left_email = _normalize_email(left_text)
        right_email = _normalize_email(right_text)
        return 100.0 if left_email == right_email else _sequence_similarity(left_email, right_email)

    if "phone" in lowered:
        left_phone = _normalize_phone(left_text)
        right_phone = _normalize_phone(right_text)
        if not left_phone or not right_phone:
            return None
        if left_phone == right_phone:
            return 100.0
        if len(left_phone) >= 7 and len(right_phone) >= 7 and left_phone[-7:] == right_phone[-7:]:
            return 92.0
        return _sequence_similarity(left_phone, right_phone)

    if any(token in lowered for token in ("domain", "website", "url", "web")):
        left_domain = _normalize_domain(left_text)
        right_domain = _normalize_domain(right_text)
        if not left_domain or not right_domain:
            return None
        if left_domain == right_domain:
            return 100.0

        left_parts = left_domain.split(".")
        right_parts = right_domain.split(".")
        left_registrable = ".".join(left_parts[-2:]) if len(left_parts) >= 2 else left_domain
        right_registrable = ".".join(right_parts[-2:]) if len(right_parts) >= 2 else right_domain
        if left_registrable == right_registrable:
            return 95.0
        return _sequence_similarity(left_domain, right_domain)

    left_norm = _normalize_text(left_text)
    right_norm = _normalize_text(right_text)
    if not left_norm or not right_norm:
        return None
    return _sequence_similarity(left_norm, right_norm)


def _row_similarity(left: Dict[str, Any], right: Dict[str, Any], key_columns: Sequence[str]) -> float:
    scores: List[float] = []
    has_exact_anchor = False
    has_domain_anchor = False

    for column in key_columns:
        score = _column_similarity(column, left.get(column), right.get(column))
        if score is None:
            continue
        scores.append(score)
        lowered = column.lower()
        if "email" in lowered and score >= 100.0:
            has_exact_anchor = True
        elif "phone" in lowered and score >= 100.0:
            has_exact_anchor = True
        elif any(token in lowered for token in ("domain", "website", "url", "web")) and score >= 95.0:
            has_domain_anchor = True

    if not scores:
        return 0.0

    blended = sum(scores) / len(scores)
    if has_exact_anchor:
        return max(blended, 98.0)
    if has_domain_anchor and left.get("__match_name") and left.get("__match_name") == right.get("__match_name"):
        return max(blended, 90.0)
    return blended


def _choose_golden(members: Sequence[Dict[str, Any]], survivor_policy: str) -> int:
    best_index = 0
    best_score = float("-inf")
    for index, row in enumerate(members):
        score = 0.0
        for key, value in row.items():
            if key.startswith("__"):
                continue
            if str(value or "").strip():
                score += 1.0
        if row.get("__display_email"):
            score += 4.0
        if row.get("__display_phone"):
            score += 2.5
        if row.get("__display_domain"):
            score += 2.0
        if row.get("__display_company"):
            score += 1.0
        if survivor_policy == "crm_person":
            object_type = str(row.get("__object_type", "")).strip().lower()
            if object_type == "contact":
                score += 8.0
            elif object_type == "lead":
                score += 3.0
        score += 0.001 * index
        if score > best_score:
            best_score = score
            best_index = index
    return best_index


def _cluster_rows(
    rows: Sequence[Dict[str, Any]],
    key_columns: Sequence[str],
    threshold: int,
    survivor_policy: str,
) -> List[Cluster]:
    union_find = _UnionFind(range(len(rows)))

    for left_index in range(len(rows)):
        for right_index in range(left_index + 1, len(rows)):
            similarity = _row_similarity(rows[left_index], rows[right_index], key_columns)
            if similarity >= threshold:
                union_find.union(left_index, right_index)

    groups: Dict[int, List[Dict[str, Any]]] = defaultdict(list)
    for index, row in enumerate(rows):
        groups[union_find.find(index)].append(row)

    clusters: List[Cluster] = []
    for members in groups.values():
        golden_index = _choose_golden(members, survivor_policy=survivor_policy)
        clusters.append(Cluster(members=list(members), golden_index=golden_index, merge_confidence=1.0))
    return clusters


def _extract_pairs(matches_df: Any, id_field: str) -> List[Tuple[str, str]]:
    try:
        import pandas as pd  # type: ignore
    except Exception as exc:  # pragma: no cover - optional dependency
        raise RuntimeError("pandas is required for engine dedup.") from exc

    pairs: List[Tuple[str, str]] = []
    columns = list(getattr(matches_df, "columns", []))
    id_like = [column for column in columns if "id" in str(column).lower()]

    def _append_pairs(left_column: str, right_column: str) -> None:
        for _, row in matches_df.iterrows():
            left_value = row.get(left_column)
            right_value = row.get(right_column)
            if pd.isna(left_value) or pd.isna(right_value):
                continue
            left_text = str(left_value).strip()
            right_text = str(right_value).strip()
            if not left_text or not right_text or left_text == right_text:
                continue
            pairs.append((left_text, right_text))

    if id_field in columns and len(id_like) >= 2:
        for column in id_like:
            if column == id_field:
                continue
            _append_pairs(id_field, column)
        if pairs:
            return pairs

    if len(id_like) >= 2:
        _append_pairs(id_like[0], id_like[1])

    return pairs


def _clusters_from_pairs(
    rows: Sequence[Dict[str, Any]],
    pairs: Sequence[Tuple[str, str]],
    survivor_policy: str,
) -> List[Cluster]:
    id_to_row = {str(row["__row_index"]): row for row in rows}
    union_find = _UnionFind(id_to_row)

    for left, right in pairs:
        if left in id_to_row and right in id_to_row:
            union_find.union(left, right)

    groups: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
    for row_id, row in id_to_row.items():
        groups[union_find.find(row_id)].append(row)

    clusters: List[Cluster] = []
    for members in groups.values():
        golden_index = _choose_golden(members, survivor_policy=survivor_policy)
        clusters.append(Cluster(members=list(members), golden_index=golden_index, merge_confidence=1.0))
    return clusters


def _build_engine_mappings(key_columns: Sequence[str]) -> List[Dict[str, Any]]:
    mappings: List[Dict[str, Any]] = []
    for column in key_columns:
        field_name = column.lower()
        mapping: Dict[str, Any] = {"source": column, "reference": column, "weight": 1.0}
        if "email" in field_name:
            mapping["algorithms"] = [
                {"name": "Exact", "weight": 0.5},
                {"name": "JaroWinkler", "weight": 0.3},
                {"name": "WRatio", "weight": 0.2},
            ]
        elif "phone" in field_name:
            mapping["algorithms"] = [
                {"name": "Exact", "weight": 0.6},
                {"name": "JaroWinkler", "weight": 0.4},
            ]
        elif any(token in field_name for token in ("domain", "website", "url", "web")):
            mapping["algorithms"] = [
                {"name": "TokenSetRatio", "weight": 0.5},
                {"name": "WRatio", "weight": 0.3},
                {"name": "JaroWinkler", "weight": 0.2},
            ]
        elif any(token in field_name for token in ("name", "company", "account", "org")):
            mapping["algorithms"] = [
                {"name": "WRatio", "weight": 0.5},
                {"name": "TokenSetRatio", "weight": 0.3},
                {"name": "PartialRatio", "weight": 0.2},
            ]
        else:
            mapping["algorithms"] = [
                {"name": "WRatio", "weight": 0.6},
                {"name": "TokenSetRatio", "weight": 0.4},
            ]
        mapping["preferred_algo"] = mapping["algorithms"][0]["name"]
        mappings.append(mapping)
    return mappings


def _engine_cluster(
    rows: Sequence[Dict[str, Any]],
    key_columns: Sequence[str],
    threshold: int,
    survivor_policy: str,
) -> List[Cluster]:
    try:
        import pandas as pd  # type: ignore
        from fmatch.saas.services.match_gateway import MatchGateway  # type: ignore
    except Exception as exc:  # pragma: no cover - optional dependency
        raise RuntimeError("MatchGateway is not available") from exc

    df = pd.DataFrame(rows)
    id_field = "__source_id"
    df[id_field] = [str(row["__row_index"]) for row in rows]
    mappings = _build_engine_mappings(key_columns)

    async def _run() -> Any:
        gateway = await MatchGateway.create()
        return await gateway.dedup(df, threshold=threshold, mappings=mappings)

    try:
        result = asyncio.run(_run())
    except Exception as exc:  # pragma: no cover - runtime depends on engine env
        raise RuntimeError(f"Gateway dedup failed: {exc}") from exc

    matches_df = getattr(result, "matches", None)
    if matches_df is None:
        raise RuntimeError("Engine dedup did not return matches DataFrame.")

    pairs = _extract_pairs(matches_df, id_field)
    return _clusters_from_pairs(rows, pairs, survivor_policy=survivor_policy)


def _recommended_action(members: Sequence[Dict[str, Any]]) -> str:
    object_types = {str(member.get("__object_type", "")).strip().lower() for member in members}
    if "lead" in object_types and "contact" in object_types:
        return "lead_to_contact_resolution"
    return "merge"


def _cluster_type(members: Sequence[Dict[str, Any]]) -> str:
    object_types = {str(member.get("__object_type", "")).strip().lower() for member in members}
    if len(object_types) <= 1:
        return "same_object"
    if "lead" in object_types and "contact" in object_types:
        return "lead_to_contact"
    return "cross_object"


def _estimate_confidence(anchors: Sequence[str], size: int) -> float:
    confidence = 0.8
    if "email" in anchors:
        confidence = 0.99
    elif "phone" in anchors:
        confidence = 0.95
    elif "domain_name" in anchors:
        confidence = 0.88
    elif "company_name" in anchors:
        confidence = 0.84

    if size >= 3 and confidence < 0.99:
        confidence = min(confidence + 0.02, 0.99)
    return round(confidence, 2)


def _has_low_name_similarity(anchor_record: Dict[str, Any], members: Sequence[Dict[str, Any]]) -> bool:
    anchor_name = str(anchor_record.get("__match_name", "") or "").strip()
    if not anchor_name:
        return False
    for member in members:
        if member is anchor_record:
            continue
        member_name = str(member.get("__match_name", "") or "").strip()
        if not member_name:
            continue
        if _sequence_similarity(anchor_name, member_name) < 55.0:
            return True
    return False


def _review_notes(
    members: Sequence[Dict[str, Any]],
    anchors: Sequence[str],
    *,
    golden_record: Dict[str, Any],
) -> List[str]:
    notes: List[str] = []
    action = _recommended_action(members)
    if action == "lead_to_contact_resolution":
        notes.append(
            "Mixed Lead + Contact cluster; resolve with Lead-to-Contact review instead of blind merge."
        )
    if "email" in anchors and _has_low_name_similarity(golden_record, members):
        notes.append("Exact email anchor but divergent names detected; verify this is not a shared inbox.")
    if "email" not in anchors and "phone" not in anchors:
        notes.append("No exact email/phone anchor; keep survivor approval behind a human review gate.")
    if len(members) >= 5:
        notes.append("Large duplicate cluster; validate downstream ownership before execution.")
    return notes


def _public_record(row: Dict[str, Any]) -> Dict[str, Any]:
    payload = {key: value for key, value in row.items() if not key.startswith("__")}
    payload.update(
        {
            "record_id": row.get("__record_id", ""),
            "object_type": row.get("__object_type", ""),
            "source_file": row.get("__source_file", ""),
            "match_name": row.get("__display_name", ""),
            "match_company": row.get("__display_company", ""),
            "match_email": row.get("__display_email", ""),
            "match_phone": row.get("__display_phone", ""),
            "match_domain": row.get("__display_domain", ""),
        }
    )
    return payload


def _cluster_payload(
    cluster: Cluster,
    row_anchor_kinds: Dict[int, set[str]],
    survivor_policy: str,
    cluster_id: str,
) -> Dict[str, Any]:
    member_indexes = [int(member.get("__row_index", -1)) for member in cluster.members]
    anchors = sorted({kind for idx in member_indexes for kind in row_anchor_kinds.get(idx, set())})
    public_members = [_public_record(member) for member in cluster.members]
    golden_record = cluster.members[cluster.golden_index]
    return {
        "cluster_id": cluster_id,
        "cluster_type": _cluster_type(cluster.members),
        "recommended_action": _recommended_action(cluster.members),
        "survivor_policy": survivor_policy,
        "object_types": sorted({member.get("object_type", "") for member in public_members}),
        "anchors": anchors,
        "merge_confidence": _estimate_confidence(anchors, len(public_members)),
        "review_notes": _review_notes(cluster.members, anchors, golden_record=golden_record),
        "members": public_members,
        "golden_record": public_members[cluster.golden_index],
    }


def _write_review_worksheet(plan: Dict[str, Any], output_path: Path) -> Path:
    output_path.parent.mkdir(parents=True, exist_ok=True)
    fieldnames = [
        "cluster_id",
        "cluster_type",
        "recommended_action",
        "merge_confidence",
        "record_id",
        "object_type",
        "is_survivor",
        "source_file",
        "anchors",
        "match_name",
        "match_company",
        "match_email",
        "match_phone",
        "match_domain",
        "review_notes",
    ]
    with output_path.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=fieldnames)
        writer.writeheader()
        for cluster in plan.get("clusters", []):
            survivor_id = str(cluster.get("golden_record", {}).get("record_id", ""))
            anchors = ",".join(cluster.get("anchors", []))
            notes = " | ".join(cluster.get("review_notes", []))
            for member in cluster.get("members", []):
                writer.writerow(
                    {
                        "cluster_id": cluster.get("cluster_id", ""),
                        "cluster_type": cluster.get("cluster_type", ""),
                        "recommended_action": cluster.get("recommended_action", ""),
                        "merge_confidence": cluster.get("merge_confidence", ""),
                        "record_id": member.get("record_id", ""),
                        "object_type": member.get("object_type", ""),
                        "is_survivor": str(member.get("record_id", "")) == survivor_id,
                        "source_file": member.get("source_file", ""),
                        "anchors": anchors,
                        "match_name": member.get("match_name", ""),
                        "match_company": member.get("match_company", ""),
                        "match_email": member.get("match_email", ""),
                        "match_phone": member.get("match_phone", ""),
                        "match_domain": member.get("match_domain", ""),
                        "review_notes": notes,
                    }
                )
    return output_path


def build_enterprise_dedupe_plan(
    source_specs: Sequence[Tuple[str, Path] | SourceSpec],
    *,
    key_columns: Optional[Sequence[str]] = None,
    threshold: int | None = None,
    output_path: Path,
    review_output_path: Optional[Path] = None,
    overwrite: bool = False,
    local: bool = False,
    survivor_policy: str = "crm_person",
    blocking_keys: Optional[Sequence[str]] = None,
    profile: str = DEFAULT_PROFILE,
    block_size_cap: int = 5000,
    limit_per_source: int | None = None,
    pairwise_warning_threshold: int = 1500,
) -> Dict[str, Any]:
    if output_path.exists() and not overwrite:
        raise FileExistsError(f"Output {output_path} exists. Use --overwrite.")
    if review_output_path and review_output_path.exists() and not overwrite:
        raise FileExistsError(f"Output {review_output_path} exists. Use --overwrite.")

    normalized_sources = [
        source if isinstance(source, SourceSpec) else SourceSpec(object_type=source[0], csv_path=source[1])
        for source in source_specs
    ]
    if not normalized_sources:
        raise ValueError("At least one source export is required.")

    resolved_profile = _resolve_profile_settings(
        profile=profile,
        key_columns=key_columns,
        blocking_keys=blocking_keys,
        threshold=threshold,
    )
    requested_key_columns = list(resolved_profile["key_columns"])
    normalized_blocking_keys = list(resolved_profile["blocking_keys"])
    resolved_threshold = int(resolved_profile["threshold"])

    invalid_blocking_keys = sorted(set(normalized_blocking_keys) - ALLOWED_BLOCKING_KEYS)
    if invalid_blocking_keys:
        raise ValueError(f"Unsupported blocking keys: {', '.join(invalid_blocking_keys)}")
    if not normalized_blocking_keys:
        raise ValueError("At least one blocking key is required.")
    if block_size_cap < 2:
        raise ValueError("--block-size-cap must be at least 2.")
    if limit_per_source is not None and limit_per_source < 1:
        raise ValueError("--limit must be at least 1 when provided.")
    if pairwise_warning_threshold < 2:
        raise ValueError("pairwise_warning_threshold must be at least 2.")

    all_rows: List[Dict[str, Any]] = []
    sources_meta: List[Dict[str, Any]] = []
    warnings: List[str] = []

    for source in normalized_sources:
        rows, headers = _load_csv(source.csv_path, limit=limit_per_source)
        if not rows:
            raise ValueError(f"Input CSV is empty: {source.csv_path}")
        prepared_rows, source_meta = _prepare_source_rows(rows, headers, source)
        if limit_per_source is not None:
            source_meta["row_limit_applied"] = limit_per_source
        sources_meta.append(source_meta)
        all_rows.extend(prepared_rows)

    for row_index, row in enumerate(all_rows):
        row["__row_index"] = row_index

    resolved_key_columns = _resolve_key_columns(requested_key_columns, all_rows[0].keys())
    components, blocking_summary, row_anchor_kinds = _build_candidate_components(
        all_rows, normalized_blocking_keys, block_size_cap
    )

    used_modes: set[str] = set()
    duplicate_clusters: List[Cluster] = []
    engine_requested = not local

    for component in components:
        component_rows = [all_rows[index] for index in component]
        component_clusters: Optional[List[Cluster]] = None
        if engine_requested:
            try:
                component_clusters = _engine_cluster(
                    component_rows,
                    key_columns=resolved_key_columns,
                    threshold=resolved_threshold,
                    survivor_policy=survivor_policy,
                )
                used_modes.add("engine")
            except Exception as exc:
                warnings.append(f"Engine fallback on candidate block ({len(component_rows)} rows): {exc}")

        if component_clusters is None:
            if len(component_rows) > pairwise_warning_threshold:
                warnings.append(
                    "Large local candidate block encountered "
                    f"({len(component_rows)} rows). Tighten blocking keys or use engine mode for this segment."
                )
            component_clusters = _cluster_rows(
                component_rows,
                key_columns=resolved_key_columns,
                threshold=resolved_threshold,
                survivor_policy=survivor_policy,
            )
            used_modes.add("local")

        duplicate_clusters.extend(cluster for cluster in component_clusters if len(cluster.members) > 1)

    payload_clusters = [
        _cluster_payload(
            cluster,
            row_anchor_kinds=row_anchor_kinds,
            survivor_policy=survivor_policy,
            cluster_id=f"crm_{index:04d}",
        )
        for index, cluster in enumerate(duplicate_clusters, start=1)
    ]
    payload_clusters.sort(
        key=lambda cluster: (
            -float(cluster.get("merge_confidence", 0.0)),
            -len(cluster.get("members", [])),
            str(cluster.get("cluster_id", "")),
        )
    )

    if blocking_summary.get("ignored_examples"):
        warnings.append(
            "Some blocking groups were skipped because they exceeded the block-size cap. "
            "Review stats.blocking.ignored_examples before execution."
        )

    object_counts: Dict[str, int] = defaultdict(int)
    for row in all_rows:
        object_counts[str(row.get("__object_type", "") or "unknown")] += 1

    action_counts: Dict[str, int] = defaultdict(int)
    for cluster in payload_clusters:
        action_counts[str(cluster.get("recommended_action", "merge"))] += 1

    if not used_modes:
        mode_used = "local" if local else "engine"
    elif used_modes == {"engine"}:
        mode_used = "engine"
    elif used_modes == {"local"}:
        mode_used = "local"
    else:
        mode_used = "mixed"

    plan = {
        "$schema": "MergePlan/v2",
        "job_type": "crm_dedupe",
        "sources": sources_meta,
        "matching_profile": {
            "profile": resolved_profile["profile"],
            "profile_defaults_applied": dict(resolved_profile["defaults_applied"]),
            "profile_notes": list(resolved_profile["notes"]),
            "mode_requested": "local" if local else "engine",
            "mode_used": mode_used,
            "threshold": resolved_threshold,
            "candidate_strategy": "blocking_first",
            "key_columns": requested_key_columns,
            "resolved_key_columns": resolved_key_columns,
            "blocking_keys": normalized_blocking_keys,
            "limit_per_source": limit_per_source,
            "capabilities": ENGINE_CAPABILITIES,
        },
        "stats": {
            "records_total": len(all_rows),
            "candidate_rows": blocking_summary.get("candidate_rows", 0),
            "candidate_components": blocking_summary.get("candidate_components", 0),
            "largest_component": blocking_summary.get("largest_component", 0),
            "clusters_found": len(payload_clusters),
            "duplicates_identified": sum(max(len(cluster.get("members", [])) - 1, 0) for cluster in payload_clusters),
            "object_counts": dict(sorted(object_counts.items())),
            "actions": dict(sorted(action_counts.items())),
            "blocking": blocking_summary,
        },
        "warnings": warnings,
        "clusters": payload_clusters,
    }

    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(json.dumps(plan, indent=2), encoding="utf-8")

    if review_output_path:
        _write_review_worksheet(plan, review_output_path)

    return plan
