"""
Dedupe Service V3: Policy-driven duplicate detection with backend-first design.
All identity logic and guardrails live here, not in the UI.
"""

from __future__ import annotations
from typing import Any, Dict, List, Literal, Tuple, Optional
from collections import defaultdict
from difflib import SequenceMatcher
import json
import logging
import re

import pandas as pd
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession

from .base_intelligent_service import BaseIntelligentService
from .dedupe_service_intelligent import IntelligentDedupeService
from .salesforce_gateway import SalesforceGateway
from ..model_defs.dedupe_models import DedupePolicy, DEFAULT_POLICIES
from ...preprocessing.rules.phone_utils import parse_phone, phone_is_strong

log = logging.getLogger(__name__)

ObjectType = Literal["Lead", "Contact", "Account"]


class PolicyDrivenDedupeService(BaseIntelligentService):
    """
    Service that applies per-object policies to determine duplicates.
    Computes frequency maps, applies anchors/blockers/hard negatives,
    and returns structured evidence for the UI to display.
    """

    def __init__(self, db: AsyncSession, account_id: str):
        super().__init__(db, account_id)
        self.persist = IntelligentDedupeService(db, account_id)
        self._policy_cache = {}

    async def load_policy(self, obj: ObjectType) -> Dict[str, Any]:
        """Load dedupe policy for object type, with fallback to defaults."""
        cache_key = f"{self.account_id}:{obj}"
        if cache_key in self._policy_cache:
            return self._policy_cache[cache_key]

        # Try account-specific policy first, but handle missing table gracefully
        try:
            result = await self.db.execute(
                select(DedupePolicy).where(
                    DedupePolicy.account_id == self.account_id,
                    DedupePolicy.object_type == obj,
                    DedupePolicy.is_active == True,
                )
            )
            policy_row = result.scalar_one_or_none()

            if policy_row:
                policy = {
                    "policy_version": policy_row.policy_version,
                    "thresholds": policy_row.thresholds,
                    "anchors": policy_row.anchors,
                    "blockers": policy_row.blockers,
                    "hard_negatives": policy_row.hard_negatives,
                    "heuristics": policy_row.heuristics or {},
                }
            else:
                # Use default policy
                policy = DEFAULT_POLICIES.get(obj, DEFAULT_POLICIES["Lead"])
        except Exception as e:
            # Table doesn't exist or other DB error - use default policy
            log.debug(f"Could not load policy from DB: {e}, using default")
            policy = DEFAULT_POLICIES.get(obj, DEFAULT_POLICIES["Lead"])

        self._policy_cache[cache_key] = policy
        return policy

    # === Normalization helpers ===

    @staticmethod
    def norm_email(v: str | None) -> Optional[str]:
        """Normalize email with gmail dot removal and + alias handling."""
        if not v:
            return None
        v = v.strip().lower()
        if "@" not in v:
            return None
        local, domain = v.split("@", 1)
        # Remove + aliases
        local = local.split("+", 1)[0]
        # Gmail/googlemail dot removal
        if domain in {"gmail.com", "googlemail.com"}:
            local = local.replace(".", "")
        return f"{local}@{domain}"

    @staticmethod
    def email_domain(v: str | None) -> Optional[str]:
        """Extract domain from email."""
        if not v or "@" not in v:
            return None
        return v.split("@", 1)[1].lower()

    @staticmethod
    def norm_phone(v: str | None, region_hint: str = "US") -> Optional[str]:
        """Normalize phone using global phone parser."""
        if not v:
            return None
        parsed = parse_phone(v, region_hint)
        # Prefer E.164 format, fallback to national digits
        return parsed["e164"] or parsed["national"] or None

    @staticmethod
    def norm_domain(v: str | None) -> str:
        """Normalize website to domain."""
        v = (v or "").strip().lower()
        if not v:
            return ""
        v = re.sub(r"^https?://", "", v)
        v = re.sub(r"^www\.", "", v)
        v = v.split("/", 1)[0]
        return v

    @staticmethod
    def same_linkedin(a: Optional[str], b: Optional[str]) -> bool:
        """Check if two LinkedIn URLs/handles are the same."""
        if not a or not b:
            return False

        def clean(x: str) -> str:
            x = x.strip().lower()
            x = re.sub(r"^https?://", "", x)
            x = re.sub(r"^www\.", "", x)
            x = x.replace("linkedin.com/in/", "").replace("linkedin.com/pub/", "")
            x = x.replace("linkedin.com/company/", "")
            return x.strip("/")

        return clean(a) == clean(b)

    # === Heuristic helpers ===

    def is_mobile_like(
        self, rec: Dict[str, Any], num: Optional[str], policy: Dict
    ) -> bool:
        """Check if phone number appears to be mobile/personal."""
        if not num:
            return False

        config = policy.get("heuristics", {}).get("mobile_detection", {})

        # Prefer MobilePhone field
        if config.get("prefer_mobile_field", True) and rec.get("MobilePhone"):
            mobile_norm = self.norm_phone(rec.get("MobilePhone"))
            if mobile_norm and mobile_norm == num:
                return True

        # Reject extensions
        if config.get("reject_extensions", True):
            phone_raw = rec.get("Phone") or ""
            if "x" in phone_raw.lower() or "ext" in phone_raw.lower():
                return False

        # Check regional mobile prefixes (JP: 070/080/090, KR: 010)
        def _mobile_prefix(national: str) -> bool:
            return national.startswith(("070", "080", "090", "010"))

        if _mobile_prefix(num):
            return True

        # Check office patterns
        patterns = policy.get("heuristics", {}).get("office_phone_patterns", [])
        for pattern in patterns:
            if re.search(pattern, num):
                return False

        # Length check
        min_digits = config.get("min_digits", 10)
        if len(num) < min_digits:
            return False

        return True

    def phone_is_strong_local(
        self,
        rec_a: Dict,
        rec_b: Dict,
        pa: Optional[str],
        pb: Optional[str],
        phone_freq: Dict[str, int],
        policy: Dict,
    ) -> bool:
        """Use global phone_is_strong utility with policy config."""
        if not (pa and pb):
            return False

        # Check frequency threshold from policy
        max_freq = 2  # default
        for anchor in policy.get("anchors", []):
            if isinstance(anchor, dict) and "phone_exact" in anchor:
                max_freq = anchor["phone_exact"].get("max_freq", 2)
                break

        # Convert to parsed phone format for global utility
        parsed_a = {
            "e164": pa if pa.startswith("+") else "",
            "national": pa,
            "type": "UNKNOWN",
        }
        parsed_b = {
            "e164": pb if pb.startswith("+") else "",
            "national": pb,
            "type": "UNKNOWN",
        }

        # Add mobile detection based on local policy
        if self.is_mobile_like(rec_a, pa, policy):
            parsed_a["type"] = "MOBILE"
        if self.is_mobile_like(rec_b, pb, policy):
            parsed_b["type"] = "MOBILE"

        return phone_is_strong(rec_a, rec_b, parsed_a, parsed_b, phone_freq, max_freq)

    # === Core matching logic ===

    def check_anchor(
        self,
        a: Dict[str, Any],
        b: Dict[str, Any],
        obj: ObjectType,
        anchor_spec: Any,
        phone_freq: Dict[str, int],
        policy: Dict,
    ) -> Tuple[bool, str]:
        """Check if records match on a specific anchor."""

        # Extract region hint from policy for accurate phone parsing
        region_hint = (policy.get("phone", {}) or {}).get("region_hint", "US")

        # Simple string anchor
        if isinstance(anchor_spec, str):
            if anchor_spec == "email_exact":
                ea = self.norm_email(a.get("Email"))
                eb = self.norm_email(b.get("Email"))
                if ea and eb and ea == eb:
                    return True, "email"

            elif anchor_spec == "linkedin_exact":
                if "LinkedIn__c" in a and "LinkedIn__c" in b:
                    if self.same_linkedin(a.get("LinkedIn__c"), b.get("LinkedIn__c")):
                        return True, "linkedin"

            elif anchor_spec == "external_id_exact":
                if "External_Id__c" in a and "External_Id__c" in b:
                    if a.get("External_Id__c") and a["External_Id__c"] == b.get(
                        "External_Id__c"
                    ):
                        return True, "external_id"

            elif anchor_spec == "vendor_lead_id_exact":
                if "Vendor_Lead_Id__c" in a and "Vendor_Lead_Id__c" in b:
                    if a.get("Vendor_Lead_Id__c") and a["Vendor_Lead_Id__c"] == b.get(
                        "Vendor_Lead_Id__c"
                    ):
                        return True, "vendor_id"

            elif anchor_spec == "registration_id_exact":
                for field in policy.get("heuristics", {}).get(
                    "registration_fields", ["EIN__c", "ABN__c"]
                ):
                    if field in a and field in b:
                        if a.get(field) and a[field] == b.get(field):
                            return True, "registration"

        # Complex anchor with conditions
        elif isinstance(anchor_spec, dict):
            if "phone_exact" in anchor_spec:
                pa = self.norm_phone(
                    a.get("Phone") or a.get("MobilePhone"), region_hint
                )
                pb = self.norm_phone(
                    b.get("Phone") or b.get("MobilePhone"), region_hint
                )
                if self.phone_is_strong_local(a, b, pa, pb, phone_freq, policy):
                    return True, "phone_strong"

            elif "account_name" in anchor_spec:
                if obj == "Contact":
                    if a.get("AccountId") and a["AccountId"] == b.get("AccountId"):
                        name_a = f"{a.get('FirstName', '')} {a.get('LastName', '')}".strip().lower()
                        name_b = f"{b.get('FirstName', '')} {b.get('LastName', '')}".strip().lower()
                        min_sim = anchor_spec["account_name"].get(
                            "name_similarity", 0.92
                        )
                        if (
                            name_a
                            and name_b
                            and SequenceMatcher(None, name_a, name_b).ratio() >= min_sim
                        ):
                            return True, "account_name"

            elif "website_name" in anchor_spec:
                if obj == "Account":
                    wa = self.norm_domain(a.get("Website"))
                    wb = self.norm_domain(b.get("Website"))
                    if wa and wb and wa == wb:
                        # Check if not free domain
                        free_domains = policy.get("heuristics", {}).get(
                            "free_domains", []
                        )
                        if (
                            anchor_spec["website_name"].get("exclude_free", True)
                            and wa in free_domains
                        ):
                            return False, ""
                        # Check name similarity
                        name_a = (a.get("Name") or "").lower()
                        name_b = (b.get("Name") or "").lower()
                        min_sim = anchor_spec["website_name"].get(
                            "name_similarity", 0.70
                        )
                        if SequenceMatcher(None, name_a, name_b).ratio() >= min_sim:
                            return True, "website"

            elif "phone_name" in anchor_spec:
                if obj == "Account":
                    pa = self.norm_phone(a.get("Phone"), region_hint)
                    pb = self.norm_phone(b.get("Phone"), region_hint)
                    if pa and pb and pa == pb:
                        name_a = (a.get("Name") or "").lower()
                        name_b = (b.get("Name") or "").lower()
                        min_sim = anchor_spec["phone_name"].get("name_similarity", 0.80)
                        if SequenceMatcher(None, name_a, name_b).ratio() >= min_sim:
                            return True, "phone_name"

        return False, ""

    def check_hard_negative(
        self,
        a: Dict[str, Any],
        b: Dict[str, Any],
        obj: ObjectType,
        neg_spec: Any,
        phone_freq: Dict[str, int],
        domain_freq: Dict[str, int],
        policy: Dict,
    ) -> Tuple[bool, str]:
        """Check if records should NOT be matched despite similarities."""

        # Extract region hint from policy for accurate phone parsing
        region_hint = (policy.get("phone", {}) or {}).get("region_hint", "US")

        ea = self.norm_email(a.get("Email"))
        eb = self.norm_email(b.get("Email"))
        pa = self.norm_phone(a.get("Phone") or a.get("MobilePhone"), region_hint)
        pb = self.norm_phone(b.get("Phone") or b.get("MobilePhone"), region_hint)

        # Simple string negative
        if isinstance(neg_spec, str):
            if neg_spec == "company_only" and obj == "Lead":
                # If only matching on company, reject
                company_a = (a.get("Company") or "").strip().lower()
                company_b = (b.get("Company") or "").strip().lower()
                if company_a and company_a == company_b:
                    # Check if they have any real anchor
                    has_anchor = False
                    for anchor in policy.get("anchors", []):
                        has_match, _ = self.check_anchor(
                            a, b, obj, anchor, phone_freq, policy
                        )
                        if has_match:
                            has_anchor = True
                            break
                    if not has_anchor:
                        return True, "company_only"

            elif neg_spec == "diff_email_without_strong_phone":
                if ea and eb and ea != eb:
                    if not self.phone_is_strong_local(a, b, pa, pb, phone_freq, policy):
                        return True, "diff_email_no_strong_phone"

            elif neg_spec == "same_company_diff_contact" and obj == "Lead":
                company_a = (a.get("Company") or "").strip().lower()
                company_b = (b.get("Company") or "").strip().lower()
                if company_a and company_a == company_b:
                    if (ea and eb and ea != eb) or (pa and pb and pa != pb):
                        return True, "same_company_diff_contact"

        # Complex negative with conditions
        elif isinstance(neg_spec, dict):
            if "huge_domain_requires_second_anchor" in neg_spec:
                da = self.email_domain(ea)
                db = self.email_domain(eb)
                if da and db and da == db:
                    max_freq = neg_spec["huge_domain_requires_second_anchor"].get(
                        "max_domain_freq", 100
                    )
                    free_domains = policy.get("heuristics", {}).get("free_domains", [])
                    if da in free_domains or domain_freq.get(da, 0) > max_freq:
                        # Need a non-domain anchor
                        anchor_count = 0
                        for anchor in policy.get("anchors", []):
                            has_match, sig = self.check_anchor(
                                a, b, obj, anchor, phone_freq, policy
                            )
                            if has_match and sig != "domain":
                                anchor_count += 1
                        if anchor_count == 0:
                            return True, "huge_domain_no_second_anchor"

        return False, ""

    def compute_frequencies(
        self, recs: List[Dict[str, Any]], region_hint: str = "US"
    ) -> Tuple[Dict[str, int], Dict[str, int]]:
        """Compute phone and domain frequencies across all records."""
        phone_freq = defaultdict(int)
        domain_freq = defaultdict(int)

        for r in recs:
            # Phone frequency
            ph = self.norm_phone(r.get("Phone") or r.get("MobilePhone"), region_hint)
            if ph:
                phone_freq[ph] += 1

            # Domain frequency
            em = self.norm_email(r.get("Email"))
            if em:
                dom = self.email_domain(em)
                if dom:
                    domain_freq[dom] += 1

        return dict(phone_freq), dict(domain_freq)

    def build_blocking_groups(
        self,
        obj: ObjectType,
        recs: List[Dict[str, Any]],
        policy: Dict,
        phone_freq: Dict[str, int],
        region_hint: str = "US",
    ) -> Dict[str, List[str]]:
        """Build blocking groups based on policy blockers."""
        groups = defaultdict(list)

        for r in recs:
            rid = r["Id"]

            for blocker in policy.get("blockers", []):
                if isinstance(blocker, str):
                    if blocker == "email":
                        em = self.norm_email(r.get("Email"))
                        if em:
                            groups[f"em:{em}"].append(rid)

                    elif blocker == "linkedin":
                        if "LinkedIn__c" in r:
                            li = r.get("LinkedIn__c", "").strip().lower()
                            if li:
                                groups[f"li:{li}"].append(rid)

                    elif blocker == "external_id":
                        if "External_Id__c" in r:
                            eid = r.get("External_Id__c", "").strip()
                            if eid:
                                groups[f"ext:{eid}"].append(rid)

                    elif blocker == "vendor_lead_id":
                        if "Vendor_Lead_Id__c" in r:
                            vid = r.get("Vendor_Lead_Id__c", "").strip()
                            if vid:
                                groups[f"ven:{vid}"].append(rid)

                    elif blocker == "account_lastname" and obj == "Contact":
                        account_id = r.get("AccountId")
                        last_name = (r.get("LastName") or "").strip().lower()
                        if account_id and last_name:
                            groups[f"acct_name:{account_id}:{last_name}"].append(rid)

                    elif blocker == "website_domain" and obj == "Account":
                        web = self.norm_domain(r.get("Website"))
                        free_domains = policy.get("heuristics", {}).get(
                            "free_domains", []
                        )
                        if web and web not in free_domains:
                            groups[f"web:{web}"].append(rid)

                    elif blocker == "registration_id" and obj == "Account":
                        for field in policy.get("heuristics", {}).get(
                            "registration_fields", ["EIN__c", "ABN__c"]
                        ):
                            if field in r:
                                reg_id = r.get(field, "").strip()
                                if reg_id:
                                    groups[f"reg:{reg_id}"].append(rid)

                    elif blocker == "phone" and obj == "Account":
                        ph = self.norm_phone(r.get("Phone"), region_hint)
                        if ph:
                            groups[f"ph:{ph}"].append(rid)

                elif isinstance(blocker, dict):
                    if "phone_rare" in blocker:
                        max_freq = blocker["phone_rare"].get("max_freq", 2)
                        ph = self.norm_phone(
                            r.get("Phone") or r.get("MobilePhone"), region_hint
                        )
                        if ph and phone_freq.get(ph, 0) <= max_freq:
                            if self.is_mobile_like(r, ph, policy):
                                groups[f"ph:{ph}"].append(rid)

        return dict(groups)

    async def run_on_salesforce(
        self,
        sf: SalesforceGateway,
        obj: ObjectType,
        *,
        limit: int | None = 5000,
        mode: Literal["intelligent", "exact", "incremental", "rebuild"] = "intelligent",
        threshold_override: float | None = None,
        save_integration=None,
    ) -> Tuple[int, int, Dict[str, Any]]:
        """
        Main entry point for dedupe detection.
        Returns: (fetched_count, saved_suggestions, meta)
        """
        # Load policy for this object type
        policy = await self.load_policy(obj)
        policy_version = policy.get("policy_version", "unknown")

        # Fetch records from Salesforce
        recs = await self.fetch_records(
            sf, obj, limit=limit, save_integration=save_integration
        )
        if not recs:
            return (
                0,
                0,
                {
                    "policy_version": policy_version,
                    "ensemble_used": False,
                    "anchors_applied": [],
                    "rules_applied": [],
                },
            )

        # Extract region hint from policy
        region_hint = (policy.get("phone", {}) or {}).get("region_hint", "US")

        # Compute frequency maps
        phone_freq, domain_freq = self.compute_frequencies(recs, region_hint)

        # Build record lookup
        rec_by_id = {r["Id"]: r for r in recs}

        # Handle rebuild mode
        if mode == "rebuild":
            await self.db.execute(
                text("""
                DELETE FROM dedupe_suggestions
                WHERE account_id = :account_id
                AND object_type = :object_type
                AND status = 'pending'
            """),
                {"account_id": self.account_id, "object_type": obj},
            )
            await self.db.commit()

        # Detect duplicates based on mode
        if mode == "exact" or mode in ["rebuild", "incremental"]:
            pairs = await self.find_exact_pairs(
                obj, recs, policy, phone_freq, domain_freq, rec_by_id, region_hint
            )
            ensemble_used = False
        else:
            # Try intelligent with fallback to exact
            try:
                pairs = await self.find_intelligent_pairs(
                    obj,
                    recs,
                    policy,
                    phone_freq,
                    domain_freq,
                    rec_by_id,
                    threshold_override,
                )
                ensemble_used = True
            except Exception as e:
                log.warning(f"Intelligent dedupe failed, using exact: {e}")
                pairs = await self.find_exact_pairs(
                    obj, recs, policy, phone_freq, domain_freq, rec_by_id, region_hint
                )
                ensemble_used = False

        # Save suggestions
        saved = await self.persist.save_suggestions(obj, pairs)

        # Build meta information
        anchors_used = set()
        rules_applied = set()
        for pair in pairs:
            if "anchor" in pair:
                anchors_used.add(pair["anchor"])
            if "rules_applied" in pair:
                rules_applied.update(pair["rules_applied"])

        meta = {
            "policy_version": policy_version,
            "ensemble_used": ensemble_used,
            "anchors_applied": list(anchors_used),
            "rules_applied": list(rules_applied),
            "fetched": len(recs),
            "suggestions_generated": saved,
        }

        # Globalization v1 observability for dedupe
        reasons_with_content = sum(
            1 for pair in pairs if pair.get("reasons") and pair["reasons"] != "{}"
        )
        log.info(
            "GlobalizationV1-Dedupe: ensemble_used=%s reasons_present=%d/%d suggestions=%d obj=%s",
            ensemble_used,
            reasons_with_content,
            len(pairs),
            saved,
            obj,
        )

        return len(recs), saved, meta

    async def find_exact_pairs(
        self,
        obj: ObjectType,
        recs: List[Dict[str, Any]],
        policy: Dict,
        phone_freq: Dict[str, int],
        domain_freq: Dict[str, int],
        rec_by_id: Dict[str, Dict],
        region_hint: str = "US",
    ) -> List[Dict[str, Any]]:
        """Find duplicate pairs using exact matching with policy rules."""

        # Build blocking groups
        groups = self.build_blocking_groups(obj, recs, policy, phone_freq, region_hint)

        pairs = []
        seen_pairs = set()

        for group_key, ids in groups.items():
            if len(ids) < 2:
                continue

            # Determine signal type from group key
            signal = group_key.split(":", 1)[0]

            for i in range(len(ids)):
                for j in range(i + 1, len(ids)):
                    pair_key = tuple(sorted([ids[i], ids[j]]))
                    if pair_key in seen_pairs:
                        continue

                    rec_a = rec_by_id[ids[i]]
                    rec_b = rec_by_id[ids[j]]

                    # Check all anchors to find match
                    anchor_matched = None
                    for anchor_spec in policy.get("anchors", []):
                        has_match, anchor_type = self.check_anchor(
                            rec_a, rec_b, obj, anchor_spec, phone_freq, policy
                        )
                        if has_match:
                            anchor_matched = anchor_type
                            break

                    if not anchor_matched:
                        continue

                    # Check hard negatives
                    rules_applied = []
                    should_reject = False
                    for neg_spec in policy.get("hard_negatives", []):
                        is_negative, rule = self.check_hard_negative(
                            rec_a, rec_b, obj, neg_spec, phone_freq, domain_freq, policy
                        )
                        if is_negative:
                            should_reject = True
                            rules_applied.append(f"rejected:{rule}")
                            break

                    if should_reject:
                        continue

                    seen_pairs.add(pair_key)

                    # Build evidence
                    evidence = self.build_evidence(rec_a, rec_b, obj)

                    pairs.append(
                        {
                            "left_id": ids[i],
                            "right_id": ids[j],
                            "score": 1.0,
                            "tier": "CERTAIN",
                            "anchor": anchor_matched,
                            "anchors_applied": [anchor_matched],
                            "rules_applied": rules_applied,
                            "policy_version": policy.get("policy_version", "unknown"),
                            "reasons": json.dumps(evidence["reasons"])
                            if isinstance(evidence["reasons"], dict)
                            else evidence["reasons"],
                            "field_score_details": json.dumps(evidence["details"]),
                            "evidence_count": evidence["count"],
                            "evidence_strength": evidence["strength"],
                            "evidence_flags": evidence["flags"],
                            "explanation": f"Exact match on {anchor_matched}",
                        }
                    )

        return pairs

    async def find_intelligent_pairs(
        self,
        obj: ObjectType,
        recs: List[Dict[str, Any]],
        policy: Dict,
        phone_freq: Dict[str, int],
        domain_freq: Dict[str, int],
        rec_by_id: Dict[str, Dict],
        threshold_override: float | None,
    ) -> List[Dict[str, Any]]:
        """Find duplicate pairs using intelligent matching with policy rules."""

        # Convert to DataFrame for engine
        df = pd.DataFrame(recs) if recs else pd.DataFrame()

        # Configure engine
        cfg = await self.auto_configure(df, df, goal="dedupe")
        cfg = dict(cfg or {})
        cfg["return_field_details"] = True
        cfg["explain_scores"] = True
        cfg["collect_reasons"] = True

        # Call engine
        engine_res = await self.dedupe_with_engine(df, cfg)

        # Process engine results with policy filters
        pairs = []
        thresholds = policy.get("thresholds", {})
        suggest_threshold = threshold_override or thresholds.get("suggest", 0.60)

        for s in engine_res or []:
            id1 = s.get("id1") or s.get("left_id")
            id2 = s.get("id2") or s.get("right_id")
            if not id1 or not id2 or id1 == id2:
                continue

            score = float(s.get("score", 0.0))
            if score < suggest_threshold:
                continue

            rec_a = rec_by_id.get(id1)
            rec_b = rec_by_id.get(id2)
            if not rec_a or not rec_b:
                continue

            # Apply policy anchors
            anchor_matched = None
            for anchor_spec in policy.get("anchors", []):
                has_match, anchor_type = self.check_anchor(
                    rec_a, rec_b, obj, anchor_spec, phone_freq, policy
                )
                if has_match:
                    anchor_matched = anchor_type
                    break

            # Even with high score, need an anchor
            if not anchor_matched:
                continue

            # Check hard negatives
            rules_applied = []
            should_reject = False
            for neg_spec in policy.get("hard_negatives", []):
                is_negative, rule = self.check_hard_negative(
                    rec_a, rec_b, obj, neg_spec, phone_freq, domain_freq, policy
                )
                if is_negative:
                    should_reject = True
                    rules_applied.append(f"rejected:{rule}")
                    break

            if should_reject:
                continue

            # Determine tier
            auto_link = thresholds.get("auto_link", 0.95)
            if score >= auto_link:
                tier = "CERTAIN"
            elif score >= max(suggest_threshold, 0.80):
                tier = "LIKELY"
            else:
                tier = "POSSIBLE"

            # Build evidence from engine details
            evidence = self.build_evidence_from_engine(s, rec_a, rec_b)

            pairs.append(
                {
                    "left_id": id1,
                    "right_id": id2,
                    "score": score,
                    "tier": tier,
                    "anchor": anchor_matched,
                    "anchors_applied": [anchor_matched],
                    "rules_applied": rules_applied,
                    "policy_version": policy.get("policy_version", "unknown"),
                    "reasons": json.dumps(evidence["reasons"])
                    if isinstance(evidence["reasons"], dict)
                    else evidence["reasons"],
                    "field_score_details": s.get("field_score_details", "{}"),
                    "evidence_count": evidence["count"],
                    "evidence_strength": evidence["strength"],
                    "evidence_flags": evidence["flags"],
                    "ensemble_used": True,
                    "explanation": s.get("explanation", ""),
                }
            )

        return pairs

    def build_evidence(self, a: Dict, b: Dict, obj: ObjectType) -> Dict[str, Any]:
        """Build evidence structure for exact matches."""
        details = []
        reasons = {}
        flags = []

        # Check email match
        ea = self.norm_email(a.get("Email"))
        eb = self.norm_email(b.get("Email"))
        if ea and eb and ea == eb:
            details.append(
                {
                    "source_column": "Email",
                    "reference_column": "Email",
                    "score": 1.0,
                    "weight": 1.0,
                    "algorithm": "Exact",
                }
            )
            reasons["Email->Email"] = 100.0
            flags.append("email_exact")

        # Check name similarity
        if obj in ("Lead", "Contact"):
            name_a = f"{a.get('FirstName', '')} {a.get('LastName', '')}".strip()
            name_b = f"{b.get('FirstName', '')} {b.get('LastName', '')}".strip()
            if name_a and name_b:
                sim = SequenceMatcher(None, name_a, name_b).ratio()
                if sim >= 0.80:
                    details.append(
                        {
                            "source_column": "Name",
                            "reference_column": "Name",
                            "score": sim,
                            "weight": 0.5,
                            "algorithm": "SequenceMatcher",
                        }
                    )
                    reasons["Name->Name"] = round(sim * 100, 1)
                    if sim >= 0.95:
                        flags.append("name_strong")

        # Check company/account
        if obj == "Lead":
            comp_a = (a.get("Company") or "").strip()
            comp_b = (b.get("Company") or "").strip()
            if comp_a and comp_b and comp_a.lower() == comp_b.lower():
                details.append(
                    {
                        "source_column": "Company",
                        "reference_column": "Company",
                        "score": 1.0,
                        "weight": 0.3,
                        "algorithm": "Exact",
                    }
                )
                reasons["Company->Company"] = 100.0

        evidence_count = len(details)
        evidence_strength = sum(d["score"] * d["weight"] for d in details) / max(
            1, sum(d["weight"] for d in details)
        )

        return {
            "details": details,
            "reasons": reasons,
            "count": evidence_count,
            "strength": evidence_strength,
            "flags": flags,
        }

    def build_evidence_from_engine(
        self, engine_result: Dict, a: Dict, b: Dict
    ) -> Dict[str, Any]:
        """Build evidence structure from engine results."""
        details = engine_result.get("details", [])
        reasons = {}
        flags = []

        # Parse details to build reasons
        for detail in details:
            if isinstance(detail, dict):
                src = detail.get("source_column", "?")
                ref = detail.get("reference_column", src)
                score = float(detail.get("score", 0))
                key = f"{src}->{ref}"
                reasons[key] = round(score * 100, 1)

                # Set flags based on fields and scores
                if src.lower() == "email" and score >= 0.99:
                    flags.append("email_exact")
                elif src.lower() in ("name", "firstname", "lastname") and score >= 0.95:
                    flags.append("name_strong")
                elif src.lower() in ("company", "account") and score >= 0.90:
                    flags.append("company_match")

        # Add ensemble flag if multiple algorithms used
        if "ensemble" in engine_result.get("explanation", "").lower():
            flags.append("ensemble_signals")

        evidence_count = len(details)
        evidence_strength = engine_result.get("evidence_strength", 0.0)
        if not evidence_strength and details:
            # Calculate if not provided
            total_weighted = sum(
                d.get("score", 0) * d.get("weight", 1) for d in details
            )
            total_weight = sum(d.get("weight", 1) for d in details)
            evidence_strength = total_weighted / max(1, total_weight)

        return {
            "details": details,
            "reasons": reasons,
            "count": evidence_count,
            "strength": evidence_strength,
            "flags": flags,
        }

    async def fetch_records(
        self, sf, obj: ObjectType, limit: int | None = 5000, save_integration=None
    ) -> List[Dict[str, Any]]:
        """Fetch records from Salesforce with available fields."""
        # Get available fields
        fields = await self.get_available_fields(sf, obj)

        # Build SOQL query
        where_clause = "IsDeleted = false"
        if obj == "Lead":
            where_clause += " AND IsConverted = false"

        soql = f"SELECT {', '.join(fields)} FROM {obj} WHERE {where_clause}"
        if limit:
            soql += f" LIMIT {int(limit)}"

        data = await sf.soql(soql)
        return data.get("records", [])

    async def get_available_fields(self, sf, obj: ObjectType) -> List[str]:
        """Get the list of fields available for this object type."""
        cache_key = f"fields_{obj}"
        if hasattr(self, cache_key):
            return getattr(self, cache_key)

        # Core fields that should always exist
        core_fields = {
            "Lead": [
                "Id",
                "FirstName",
                "LastName",
                "Company",
                "Email",
                "Phone",
                "MobilePhone",
                "CreatedDate",
                "IsConverted",
                "IsDeleted",
            ],
            "Contact": [
                "Id",
                "FirstName",
                "LastName",
                "AccountId",
                "Email",
                "Phone",
                "MobilePhone",
                "CreatedDate",
                "IsDeleted",
            ],
            "Account": ["Id", "Name", "Website", "Phone", "CreatedDate", "IsDeleted"],
        }

        fields = list(core_fields.get(obj, []))

        # Try to add custom fields if they exist
        try:
            describe_result = await (
                sf.describe_object(obj)
                if hasattr(sf, "describe_object")
                else sf.describe(obj)
            )
            available_field_names = {
                f["name"] for f in describe_result.get("fields", [])
            }

            # Optional fields to check
            optional = {
                "Lead": ["LinkedIn__c", "External_Id__c", "Vendor_Lead_Id__c"],
                "Contact": ["LinkedIn__c", "External_Id__c"],
                "Account": ["LinkedIn_Company__c", "EIN__c", "ABN__c", "VAT__c"],
            }

            for field in optional.get(obj, []):
                if field in available_field_names:
                    fields.append(field)
        except Exception as e:
            log.debug(f"Could not describe {obj}: {e}")

        # Cache for this session
        setattr(self, cache_key, fields)
        return fields
