# src/fmatch/saas/services/discovery_agent.py
"""
Field Discovery Agent - Hybrid deterministic + local AI approach.
Automatically discovers the best matching fields for any Salesforce org.
Privacy-first: Only processes schema metadata, never actual record values.
"""

from typing import Dict, List, Optional, Any
from dataclasses import dataclass
from datetime import datetime, timedelta
import json
import logging
import hashlib
from collections import Counter
import re
from enum import Enum

import numpy as np
import httpx

from .utils.data_profile import normalize_scalar

logger = logging.getLogger(__name__)


# Field semantic classes for LLM classification
class FieldClass(str, Enum):
    UNIQUE_IDENTIFIER = "UNIQUE_IDENTIFIER"
    EMAIL = "EMAIL"
    PHONE = "PHONE"
    WEBSITE_DOMAIN = "WEBSITE_DOMAIN"
    COMPANY_NAME = "COMPANY_NAME"
    PERSON_NAME = "PERSON_NAME"
    ADDRESS_LINE = "ADDRESS_LINE"
    POSTAL_CODE = "POSTAL_CODE"
    GEOGRAPHIC = "GEOGRAPHIC"
    INDUSTRY = "INDUSTRY"
    REFERENCE_ID = "REFERENCE_ID"
    GENERIC_TEXT = "GENERIC_TEXT"
    IGNORE = "IGNORE"


# Multipliers for each semantic class
CLASS_MULTIPLIERS = {
    FieldClass.UNIQUE_IDENTIFIER: 2.0,
    FieldClass.EMAIL: 1.5,
    FieldClass.WEBSITE_DOMAIN: 1.5,
    FieldClass.PHONE: 1.3,
    FieldClass.COMPANY_NAME: 1.3,
    FieldClass.POSTAL_CODE: 1.2,
    FieldClass.PERSON_NAME: 1.1,
    FieldClass.REFERENCE_ID: 1.4,
    FieldClass.ADDRESS_LINE: 1.0,
    FieldClass.GEOGRAPHIC: 0.8,
    FieldClass.INDUSTRY: 0.8,
    FieldClass.GENERIC_TEXT: 0.6,
    FieldClass.IGNORE: 0.0,
}


@dataclass
class FieldProfile:
    """Complete profile of a field combining heuristics and semantics."""

    # Basic metadata
    api_name: str
    label: str
    field_type: str
    help_text: Optional[str] = None

    # Salesforce metadata flags
    is_external_id: bool = False
    is_unique: bool = False
    is_indexed: bool = False
    is_required: bool = False
    is_custom: bool = False

    # Heuristic statistics
    fill_rate: float = 0.0
    distinct_rate: float = 0.0
    format_valid_rate: float = 0.0
    entropy: float = 0.0
    avg_length: float = 0.0
    null_rate: float = 0.0

    # Special indicators
    personal_email_rate: float = 0.0  # For email fields
    stability_score: float = 0.0  # How stable values are

    # Scores
    heuristic_score: float = 0.0
    semantic_class: Optional[FieldClass] = None
    semantic_confidence: float = 0.0
    semantic_rationale: Optional[str] = None
    final_score: float = 0.0

    # Matching configuration
    comparator: str = "fuzzy"  # exact, domain, fuzzy, phonetic
    blocking_key: Optional[str] = None
    weight: float = 0.0


@dataclass
class DiscoveryPolicy:
    """Discovered matching policy for an object."""

    object_type: str
    discovered_at: datetime
    sample_size: int
    selected_fields: List[FieldProfile]
    blocking_keys: List[str]
    weights: Dict[str, float]
    thresholds: Dict[str, float]
    version: int = 1
    cache_key: Optional[str] = None


class DiscoveryAgent:
    """
    Discovers optimal matching fields using hybrid approach:
    1. Deterministic profiling of all fields
    2. Local LLM for semantic classification (schema-only)
    3. Policy synthesis with explainable weights
    """

    # Regex patterns for field validation
    EMAIL_PATTERN = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
    PHONE_PATTERN = re.compile(r"^[\d\s\(\)\-\+\.]+$")
    DOMAIN_PATTERN = re.compile(
        r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$"
    )
    POSTAL_PATTERN = re.compile(
        r"^\d{5}(?:-\d{4})?$|^[A-Z]\d[A-Z]\s?\d[A-Z]\d$"
    )  # US/Canada

    # Personal email domains
    PERSONAL_DOMAINS = {
        "gmail.com",
        "yahoo.com",
        "hotmail.com",
        "outlook.com",
        "aol.com",
        "icloud.com",
        "me.com",
        "mac.com",
        "live.com",
    }

    def __init__(
        self,
        llm_endpoint: Optional[str] = None,
        use_ai: bool = True,
        cache_ttl_days: int = 30,
    ):
        self.llm_endpoint = (
            llm_endpoint or "http://localhost:8080/v1/completions"
        )  # Local vLLM
        self.use_ai = use_ai
        self.cache_ttl = timedelta(days=cache_ttl_days)
        self._cache = {}  # In-memory cache for session

    async def discover(
        self,
        sf_client,
        object_type: str,
        sample_size: int = 1000,
        max_fields: int = 100,
        force_refresh: bool = False,
    ) -> DiscoveryPolicy:
        """
        Main entry point - discovers optimal fields for matching.

        Args:
            sf_client: Salesforce client
            object_type: Account, Contact, or Lead
            sample_size: Number of records to sample
            max_fields: Maximum fields to analyze (SOQL limit)
            force_refresh: Bypass cache

        Returns:
            DiscoveryPolicy with selected fields and weights
        """

        # Check cache unless forced refresh
        cache_key = self._generate_cache_key(sf_client.instance_url, object_type)
        if not force_refresh and cache_key in self._cache:
            logger.info(f"Using cached policy for {object_type}")
            return self._cache[cache_key]

        logger.info(f"Starting field discovery for {object_type}")

        try:
            # Phase 1: Get schema and sample data
            schema = await sf_client.describe_object(object_type)
            fields_meta = schema.get("fields", [])

            # Phase 2: Heuristic profiling (deterministic, fast)
            profiles = await self._profile_all_fields(
                sf_client, object_type, fields_meta, sample_size, max_fields
            )

            # Phase 3: Semantic classification (AI, schema-only)
            if self.use_ai:
                profiles = await self._classify_with_llm(profiles)

            # Phase 4: Policy synthesis
            policy = self._synthesize_policy(object_type, profiles, sample_size)

            # Cache the policy
            self._cache[cache_key] = policy

            logger.info(
                f"Discovery complete: {len(policy.selected_fields)} fields selected "
                f"from {len(profiles)} analyzed"
            )

            return policy

        except Exception as e:
            logger.error(f"Discovery failed for {object_type}: {e}")
            # Fall back to hardcoded defaults
            return self._get_fallback_policy(object_type)

    async def _profile_all_fields(
        self,
        sf_client,
        object_type: str,
        fields_meta: List[Dict],
        sample_size: int,
        max_fields: int,
    ) -> List[FieldProfile]:
        """
        Layer 1: Deterministic profiling of all fields.
        Computes statistics without any AI involvement.
        """
        profiles = []

        # Filter to fields we can actually match on
        matchable_fields = [
            f
            for f in fields_meta
            if f["type"]
            in ["string", "email", "phone", "url", "textarea", "id", "reference"]
            and not f["name"].endswith("__pc")  # Skip person account fields
            and f["name"]
            not in ["Id", "CreatedDate", "LastModifiedDate", "SystemModstamp"]
        ]

        # Prioritize fields for sampling (can't query all due to SOQL limits)
        priority_fields = self._prioritize_fields_for_sampling(
            matchable_fields, max_fields
        )

        # Sample records with selected fields
        field_names = [f["name"] for f in priority_fields]
        sample = await sf_client.get_sample_records(
            object_type, field_names, n=sample_size
        )

        if not sample:
            logger.warning(f"No sample records for {object_type}")
            return []

        # Profile each field
        for field_meta in priority_fields:
            profile = self._compute_field_profile(field_meta, sample)
            profiles.append(profile)

        # Sort by heuristic score
        profiles.sort(key=lambda p: p.heuristic_score, reverse=True)

        return profiles

    def _compute_field_profile(
        self, field_meta: Dict, sample: List[Dict]
    ) -> FieldProfile:
        """
        Compute comprehensive statistics for a single field.
        This is pure deterministic analysis - no AI.
        """
        api_name = field_meta["name"]
        values = [normalize_scalar(record.get(api_name)) for record in sample]

        # Basic counts
        non_null = [v for v in values if v is not None]
        unique_values = set(non_null)

        # Create profile
        profile = FieldProfile(
            api_name=api_name,
            label=field_meta.get("label", api_name),
            field_type=field_meta.get("type", "string"),
            help_text=field_meta.get("inlineHelpText"),
            is_external_id=field_meta.get("externalId", False),
            is_unique=field_meta.get("unique", False),
            is_indexed=field_meta.get("indexed", False),
            is_required=field_meta.get("required", False),
            is_custom="__c" in api_name,
        )

        # Compute statistics
        total = len(values)
        if total > 0:
            profile.fill_rate = len(non_null) / total
            profile.null_rate = 1 - profile.fill_rate

        if non_null:
            profile.distinct_rate = len(unique_values) / len(non_null)
            profile.avg_length = np.mean([len(v) for v in non_null])

            # Compute entropy (information content)
            if len(unique_values) > 1:
                counts = Counter(non_null)
                probs = np.array(list(counts.values())) / len(non_null)
                profile.entropy = -np.sum(probs * np.log2(probs + 1e-10))

            # Field-specific validation
            profile.format_valid_rate = self._compute_format_validity(
                field_meta, non_null, profile
            )

            # Stability score (are values consistent/stable?)
            profile.stability_score = self._compute_stability(non_null)

        # Compute heuristic score
        profile.heuristic_score = self._compute_heuristic_score(profile)

        # Determine comparator
        profile.comparator = self._determine_comparator(field_meta, profile)

        return profile

    def _compute_format_validity(
        self, field_meta: Dict, values: List[Any], profile: FieldProfile
    ) -> float:
        """Check format validity based on field type."""
        if not values:
            return 0.0

        field_type = field_meta.get("type", "").lower()
        api_name = field_meta["name"].lower()

        valid_count = 0

        for value in values:
            str_val = str(value).strip()

            if field_type == "email" or "email" in api_name:
                if self.EMAIL_PATTERN.match(str_val):
                    valid_count += 1
                    # Track personal emails
                    domain = str_val.split("@")[-1].lower()
                    if domain in self.PERSONAL_DOMAINS:
                        profile.personal_email_rate += 1

            elif field_type == "phone" or "phone" in api_name or "mobile" in api_name:
                if (
                    self.PHONE_PATTERN.match(str_val)
                    and len(re.sub(r"\D", "", str_val)) >= 10
                ):
                    valid_count += 1

            elif field_type == "url" or "website" in api_name or "url" in api_name:
                if "." in str_val and len(str_val) > 4:
                    valid_count += 1

            elif "postal" in api_name or "zip" in api_name:
                if self.POSTAL_PATTERN.match(str_val):
                    valid_count += 1

            else:
                # Generic text validation - just check it's not garbage
                if len(str_val) > 1 and not str_val.isspace():
                    valid_count += 1

        if profile.personal_email_rate > 0:
            profile.personal_email_rate /= len(values)

        return valid_count / len(values)

    def _compute_stability(self, values: List[Any]) -> float:
        """
        Compute stability score - how consistent/categorical are values?
        High stability = likely categorical/picklist
        Low stability = likely free text
        """
        if len(values) < 10:
            return 0.5

        # Check if values are mostly from a small set (categorical)
        unique_ratio = len(set(values)) / len(values)

        if unique_ratio < 0.1:  # Very few unique values
            return 0.9  # High stability (categorical)
        elif unique_ratio > 0.9:  # Almost all unique
            return 0.2  # Low stability (likely IDs or free text)
        else:
            return 1 - unique_ratio  # Inverse relationship

    def _compute_heuristic_score(self, profile: FieldProfile) -> float:
        """
        Compute overall heuristic score based on statistics.
        This is the deterministic scoring - no AI involved.
        """
        score = 0.0

        # Weighted components
        score += 0.35 * profile.fill_rate  # Data availability
        score += 0.30 * profile.distinct_rate  # Uniqueness
        score += 0.20 * profile.format_valid_rate  # Quality

        # Entropy bonus (information content)
        if profile.entropy > 0:
            entropy_normalized = min(1.0, profile.entropy / 10.0)
            score += 0.15 * entropy_normalized

        # Metadata boosts
        if profile.is_external_id:
            score *= 2.0  # Huge boost for external IDs
        elif profile.is_unique:
            score *= 1.8
        elif profile.is_indexed:
            score *= 1.3

        # Penalties
        if profile.field_type == "textarea":
            score *= 0.7  # Long text is less useful
        if profile.stability_score > 0.8:
            score *= 0.8  # Too categorical
        if profile.personal_email_rate > 0.5:
            score *= 0.5  # Personal emails less useful for B2B

        return min(1.0, score)  # Cap at 1.0

    def _determine_comparator(self, field_meta: Dict, profile: FieldProfile) -> str:
        """Determine the best comparison method for this field."""
        field_type = field_meta.get("type", "").lower()
        api_name = field_meta["name"].lower()

        # Exact match fields
        if (
            profile.is_external_id
            or profile.is_unique
            or field_type in ["id", "reference"]
            or profile.distinct_rate > 0.95
        ):
            return "exact"

        # Email fields
        if field_type == "email" or "email" in api_name:
            return "exact"

        # Phone fields
        if field_type == "phone" or "phone" in api_name:
            return "phone_normalized"

        # Website/URL fields
        if field_type == "url" or "website" in api_name:
            return "domain"

        # Name fields
        if any(x in api_name for x in ["name", "company", "account"]):
            return "fuzzy"

        # Address/geographic
        if any(x in api_name for x in ["address", "street", "city", "state"]):
            return "fuzzy"

        # Postal codes
        if "postal" in api_name or "zip" in api_name:
            return "exact"

        # Default to fuzzy
        return "fuzzy"

    def _prioritize_fields_for_sampling(
        self, fields: List[Dict], max_fields: int
    ) -> List[Dict]:
        """
        Prioritize which fields to sample (SOQL has limits).
        Smart ordering to get the most valuable fields first.
        """
        # Score each field for sampling priority
        scored = []
        for f in fields:
            score = 0
            name_lower = f["name"].lower()

            # High priority patterns
            if f.get("externalId"):
                score += 100
            if f.get("unique"):
                score += 90
            if f.get("indexed"):
                score += 50

            # Field name patterns (these are usually good)
            high_value_patterns = [
                "email",
                "phone",
                "website",
                "domain",
                "account",
                "company",
                "name",
                "_id",
                "_ref",
                "_num",
                "legacy",
                "old",
            ]
            for pattern in high_value_patterns:
                if pattern in name_lower:
                    score += 30

            # Custom fields often have business value
            if "__c" in f["name"]:
                score += 20

            # Standard important fields
            if f["name"] in ["Name", "Email", "Phone", "Website"]:
                score += 80

            scored.append((score, f))

        # Sort by priority and take top N
        scored.sort(key=lambda x: x[0], reverse=True)
        return [f for _, f in scored[:max_fields]]

    async def _classify_with_llm(
        self, profiles: List[FieldProfile], batch_size: int = 20
    ) -> List[FieldProfile]:
        """
        Layer 2: Use local LLM to classify field semantics.
        Only sends schema metadata - never actual record values.
        """
        if not self.use_ai:
            return profiles

        # Filter to fields that need semantic classification
        candidates = [
            p
            for p in profiles
            if p.is_custom  # Custom fields
            or p.heuristic_score > 0.5  # Promising fields
            or self._needs_semantic_classification(p)  # Ambiguous fields
        ]

        if not candidates:
            return profiles

        logger.info(f"Classifying {len(candidates)} fields with local LLM")

        # Process in batches to avoid overwhelming local model
        for i in range(0, len(candidates), batch_size):
            batch = candidates[i : i + batch_size]

            try:
                classifications = await self._call_local_llm(batch)

                # Update profiles with semantic classifications
                for profile, classification in zip(batch, classifications):
                    if classification:
                        profile.semantic_class = classification.get("class")
                        profile.semantic_confidence = classification.get(
                            "confidence", 0.0
                        )
                        profile.semantic_rationale = classification.get("rationale")

            except Exception as e:
                logger.warning(f"LLM classification failed for batch: {e}")
                # Continue without semantic layer - heuristics still work

        return profiles

    def _needs_semantic_classification(self, profile: FieldProfile) -> bool:
        """Determine if a field needs AI classification."""
        # Already clearly identified by heuristics
        if profile.is_external_id or profile.is_unique:
            return False

        # Standard fields we understand
        if profile.api_name in ["Name", "Email", "Phone", "Website"]:
            return False

        # Custom fields with ambiguous patterns
        if profile.is_custom and profile.heuristic_score > 0.3:
            return True

        # High entropy text fields might be IDs
        if profile.entropy > 5 and profile.distinct_rate > 0.9:
            return True

        # Fields with ID-like patterns
        name_lower = profile.api_name.lower()
        if any(x in name_lower for x in ["_id", "_ref", "_num", "code", "key"]):
            return True

        return False

    async def _call_local_llm(
        self, profiles: List[FieldProfile]
    ) -> List[Optional[Dict]]:
        """
        Call local LLM with schema-only data.
        Returns classifications for each field.
        """
        # Build privacy-safe prompt with only metadata
        prompt = self._build_classification_prompt(profiles)

        try:
            async with httpx.AsyncClient(timeout=30) as client:
                response = await client.post(
                    self.llm_endpoint,
                    json={
                        "prompt": prompt,
                        "max_tokens": 500,
                        "temperature": 0.1,  # Low temperature for consistency
                        "response_format": {"type": "json_object"},
                    },
                )
                response.raise_for_status()

                # Parse structured response
                result = response.json()
                classifications = json.loads(
                    result.get("choices", [{}])[0].get("text", "{}")
                )

                # Map back to profiles
                return [
                    classifications.get("fields", {}).get(p.api_name) for p in profiles
                ]

        except Exception as e:
            logger.error(f"Local LLM call failed: {e}")
            return [None] * len(profiles)

    def _build_classification_prompt(self, profiles: List[FieldProfile]) -> str:
        """
        Build a privacy-safe prompt using only schema metadata.
        Never includes actual record values.
        """
        field_specs = []
        for p in profiles:
            # Only schema information - no actual data
            spec = {
                "api_name": p.api_name,
                "label": p.label,
                "type": p.field_type,
                "is_custom": p.is_custom,
                "is_external_id": p.is_external_id,
                "is_unique": p.is_unique,
                "statistics": {
                    "fill_rate": round(p.fill_rate, 2),
                    "unique_rate": round(p.distinct_rate, 2),
                    "format_valid": round(p.format_valid_rate, 2),
                    "entropy": round(p.entropy, 2),
                },
            }
            if p.help_text:
                spec["help_text"] = p.help_text[:100]  # Truncate long text
            field_specs.append(spec)

        prompt = f"""You are a Salesforce schema analyzer. Classify these fields by their semantic purpose for data matching.

IMPORTANT: You are ONLY seeing field definitions and statistics. No actual customer data is provided.

Fields to classify:
{json.dumps(field_specs, indent=2)}

For each field, determine its semantic class from these options:
- UNIQUE_IDENTIFIER: External IDs, system references, unique keys
- EMAIL: Email address fields
- PHONE: Phone number fields
- WEBSITE_DOMAIN: Website or domain fields
- COMPANY_NAME: Organization or account names
- PERSON_NAME: Individual names (first, last, full)
- REFERENCE_ID: Foreign keys or relationship IDs
- ADDRESS_LINE: Street addresses
- POSTAL_CODE: Zip codes or postal codes
- GEOGRAPHIC: City, state, country fields
- INDUSTRY: Industry or sector classifications
- GENERIC_TEXT: General text without specific matching value
- IGNORE: Fields not useful for matching (timestamps, counts, etc.)

Return JSON with this structure:
{{
  "fields": {{
    "<api_name>": {{
      "class": "<CLASS_NAME>",
      "confidence": 0.0-1.0,
      "rationale": "One line explanation"
    }}
  }}
}}

Focus on fields that would be valuable for identifying duplicate records.
"""
        return prompt

    def _synthesize_policy(
        self, object_type: str, profiles: List[FieldProfile], sample_size: int
    ) -> DiscoveryPolicy:
        """
        Layer 3: Combine heuristic and semantic scores to create final policy.
        This is where explainability happens.
        """
        # Calculate final scores
        for profile in profiles:
            if profile.semantic_class and profile.semantic_confidence > 0:
                # Combine heuristic and semantic signals
                multiplier = CLASS_MULTIPLIERS.get(profile.semantic_class, 1.0)
                profile.final_score = (
                    profile.heuristic_score
                    * multiplier
                    * (0.7 + 0.3 * profile.semantic_confidence)  # Blend confidence
                )
            else:
                # Heuristic only
                profile.final_score = profile.heuristic_score

        # Select top fields (with diversity)
        selected = self._select_diverse_fields(profiles, object_type)

        # Generate blocking keys
        blocking_keys = self._generate_blocking_keys(selected)

        # Normalize weights
        total_score = sum(f.final_score for f in selected)
        weights = {}
        for f in selected:
            f.weight = f.final_score / total_score if total_score > 0 else 0
            weights[f.api_name] = f.weight

        # Set thresholds based on object type
        thresholds = self._determine_thresholds(object_type, selected)

        return DiscoveryPolicy(
            object_type=object_type,
            discovered_at=datetime.utcnow(),
            sample_size=sample_size,
            selected_fields=selected,
            blocking_keys=blocking_keys,
            weights=weights,
            thresholds=thresholds,
            version=1,
        )

    def _select_diverse_fields(
        self, profiles: List[FieldProfile], object_type: str, max_fields: int = 7
    ) -> List[FieldProfile]:
        """
        Select diverse high-value fields.
        Ensures we get different types of signals.
        """
        selected = []
        seen_types = set()

        # Sort by final score
        candidates = sorted(profiles, key=lambda p: p.final_score, reverse=True)

        # Always include certain field types if available
        for profile in candidates:
            if len(selected) >= max_fields:
                break

            # Always include external IDs and unique fields
            if profile.is_external_id or profile.is_unique:
                selected.append(profile)
                seen_types.add(profile.semantic_class or profile.field_type)
                continue

            # Include if it adds diversity
            field_type = profile.semantic_class or profile.field_type

            # Skip if we already have this type (unless it's really good)
            if field_type in seen_types and profile.final_score < 0.7:
                continue

            # Skip very low scoring fields
            if profile.final_score < 0.3:
                continue

            selected.append(profile)
            seen_types.add(field_type)

        return selected[:max_fields]

    def _generate_blocking_keys(self, fields: List[FieldProfile]) -> List[str]:
        """Generate blocking keys for efficient matching."""
        blocks = []

        for field in fields:
            if field.comparator == "domain" and field.final_score > 0.5:
                blocks.append(f"domain_trigram:{field.api_name}")

            elif field.comparator == "phone_normalized" and field.final_score > 0.5:
                blocks.append(f"phone_last7:{field.api_name}")

            elif field.comparator == "exact" and field.is_external_id:
                blocks.append(f"exact:{field.api_name}")

            elif "name" in field.api_name.lower() and field.final_score > 0.4:
                blocks.append(f"name_phonetic:{field.api_name}")

        # Always add a fallback block
        if not blocks:
            blocks.append("all_records")

        return blocks[:5]  # Limit blocking keys

    def _determine_thresholds(
        self, object_type: str, fields: List[FieldProfile]
    ) -> Dict[str, float]:
        """Determine matching thresholds based on field quality."""
        # Check if we have high-quality unique identifiers
        has_unique_id = any(
            f.is_external_id
            or f.is_unique
            or f.semantic_class == FieldClass.UNIQUE_IDENTIFIER
            for f in fields
        )

        if has_unique_id:
            # We have strong identifiers - can be more aggressive
            return {"duplicate": 0.70, "likely_duplicate": 0.80, "auto_merge": 0.95}
        else:
            # Only fuzzy fields - be more conservative
            return {"duplicate": 0.75, "likely_duplicate": 0.85, "auto_merge": 0.97}

    def _generate_cache_key(self, instance_url: str, object_type: str) -> str:
        """Generate cache key for policy."""
        return hashlib.md5(f"{instance_url}:{object_type}".encode()).hexdigest()

    def _get_fallback_policy(self, object_type: str) -> DiscoveryPolicy:
        """
        Fallback policy using standard fields when discovery fails.
        This ensures the system always works.
        """
        # Standard fields by object
        standard_fields = {
            "Account": ["Name", "Website", "Phone", "BillingCity", "BillingPostalCode"],
            "Contact": ["FirstName", "LastName", "Email", "Phone", "AccountId"],
            "Lead": ["FirstName", "LastName", "Company", "Email", "Phone"],
        }

        fields = []
        for field_name in standard_fields.get(object_type, ["Name"]):
            fields.append(
                FieldProfile(
                    api_name=field_name,
                    label=field_name,
                    field_type="string",
                    heuristic_score=0.5,
                    final_score=0.5,
                    weight=0.2,
                    comparator="fuzzy",
                )
            )

        return DiscoveryPolicy(
            object_type=object_type,
            discovered_at=datetime.utcnow(),
            sample_size=0,
            selected_fields=fields,
            blocking_keys=["all_records"],
            weights={f.api_name: f.weight for f in fields},
            thresholds={
                "duplicate": 0.75,
                "likely_duplicate": 0.85,
                "auto_merge": 0.95,
            },
            version=0,  # Version 0 indicates fallback
        )

    def explain_field_selection(self, profile: FieldProfile) -> Dict[str, Any]:
        """
        Generate human-readable explanation for why a field was selected.
        This is key for trust and explainability.
        """
        explanation = {
            "field": profile.api_name,
            "label": profile.label,
            "score": round(profile.final_score, 3),
            "weight": round(profile.weight, 3),
            "reasons": [],
        }

        # Build reason list
        if profile.is_external_id:
            explanation["reasons"].append("Marked as External ID in Salesforce")
        if profile.is_unique:
            explanation["reasons"].append("Unique field constraint")
        if profile.fill_rate > 0.9:
            explanation["reasons"].append(f"{int(profile.fill_rate * 100)}% filled")
        if profile.distinct_rate > 0.9:
            explanation["reasons"].append(
                f"{int(profile.distinct_rate * 100)}% unique values"
            )
        if profile.semantic_class and profile.semantic_confidence > 0.7:
            explanation["reasons"].append(
                f"Identified as {profile.semantic_class.value} "
                f"({int(profile.semantic_confidence * 100)}% confident)"
            )
        if profile.format_valid_rate > 0.9:
            explanation["reasons"].append(
                f"{int(profile.format_valid_rate * 100)}% valid format"
            )

        # Add semantic rationale if available
        if profile.semantic_rationale:
            explanation["ai_insight"] = profile.semantic_rationale

        # Comparator explanation
        explanation["matching_method"] = {
            "exact": "Exact match required",
            "domain": "Domain matching (ignores www, protocol)",
            "fuzzy": "Fuzzy matching (handles typos)",
            "phone_normalized": "Phone matching (ignores formatting)",
        }.get(profile.comparator, "Fuzzy matching")

        return explanation

    async def detect_l2a_relationship(self, sf_client) -> Dict[str, Any]:
        """
        Detect Lead-to-Account lookup field and analyze L2A matching potential.
        This helps identify opportunities for Lead-to-Account matching.
        """
        try:
            # Find lookup field in Lead object
            lead_schema = await sf_client.describe_object("Lead")
            l2a_field = None

            for field in lead_schema.get("fields", []):
                # Check if this is a reference field that points to Account
                if field.get("type") == "reference" and "Account" in (
                    field.get("referenceTo") or []
                ):
                    l2a_field = field["name"]
                    break

            if not l2a_field:
                return {
                    "status": "no_lookup_field",
                    "message": "No Lead-to-Account lookup field found",
                }

            # Analyze fill rate for unconverted leads
            query = f"SELECT COUNT(Id), COUNT({l2a_field}) FROM Lead WHERE IsConverted = false"
            result = await sf_client.query(query)

            if result and result.get("records"):
                total_leads = result["records"][0].get("expr0", 0)
                linked_leads = result["records"][0].get("expr1", 0)
                unlinked_leads = total_leads - linked_leads
                fill_rate = linked_leads / total_leads if total_leads > 0 else 0

                return {
                    "status": "ready",
                    "lookup_field": l2a_field,
                    "total_leads": total_leads,
                    "linked_leads": linked_leads,
                    "unlinked_leads": unlinked_leads,
                    "fill_rate": round(fill_rate, 3),
                    "opportunity": f"Link {unlinked_leads:,} unmatched leads to accounts",
                    "recommendation": "High"
                    if unlinked_leads > 100
                    else "Medium"
                    if unlinked_leads > 10
                    else "Low",
                }
            else:
                return {
                    "status": "no_data",
                    "lookup_field": l2a_field,
                    "message": "Could not query lead data",
                }

        except Exception as e:
            logger.error(f"Failed to detect L2A relationship: {e}")
            return {"status": "error", "error": str(e)}
