"""
Intelligence API - Exposes all engine smarts to the web
"""

from fastapi import APIRouter, Depends, HTTPException, Body
from fastapi.responses import JSONResponse
import logging
import os
import time
import uuid
from typing import Dict, Any, List, Optional
from uuid import UUID
from logging import LoggerAdapter
from copy import deepcopy
import pandas as pd
import numpy as np
from pydantic import BaseModel
import json
import hashlib
import re

from fmatch.core.engine import (
    determine_optimal_blocking_strategy,
    auto_select_id_column,
    BlockingMode,
    BlockingWeightProfile,
    validate_blocking_candidates,
    detect_optimal_variants,
    _get_semantic_type,
    AutoMapper,
    ColumnStatsCache,
    calculate_mapping_weights as engine_calculate_mapping_weights,
)
from fmatch.core.b2c_engine_integration import clear_domain_cache
from fmatch.preprocessing.core.column_detector import ColumnDetector
from ...auth import get_current_account

from ...storage import get_storage_backend

# Import bandits for adaptive learning
from ...intelligence.bandits import ThresholdBandit, BlockingBandit
from ...intelligence.decision_logger import DecisionLogger
from ...intelligence.policy_store import PolicyStore
from ...auth_security import require_account

# Database imports - using db module instead of database
try:
    from ...db import get_session
    from sqlalchemy.ext.asyncio import AsyncSession
except ImportError:
    # Fallback if database is not configured
    get_session = None
    AsyncSession = None

router = APIRouter(prefix="/api/v2/intelligence", tags=["Intelligence"])
log = logging.getLogger(__name__)

# Module-level singleton for DecisionLogger to avoid races
_DECISION_LOGGER = None


def _get_decision_logger(redis_conn=None):
    """Get or create singleton DecisionLogger instance"""
    global _DECISION_LOGGER
    if _DECISION_LOGGER is None:
        _DECISION_LOGGER = DecisionLogger(redis_conn=redis_conn)
    return _DECISION_LOGGER


# Request/Response Models
class AnalyzeRequest(BaseModel):
    job_id: str
    sample_size: int = 10000


class BlockingAnalysisRequest(BaseModel):
    job_id: str
    weight_profile: str = "balanced"
    custom_weights: Optional[Dict[str, float]] = None


class MappingSuggestionRequest(BaseModel):
    job_id: str
    threshold: int = 75
    enable_multi_algo: bool = True


def _normalize_header(name: Any) -> str:
    return re.sub(r"[^a-z0-9]", "", str(name or "").lower())


HEADER_SYNONYMS: Dict[str, set[str]] = {
    "companyname": {"accountname", "company", "businessname", "organization", "org"},
    "accountname": {"companyname", "company", "business", "organization"},
    "company": {"companyname", "accountname", "business", "organization"},
    "business": {"company", "accountname", "companyname", "businessname"},
    "businessname": {"companyname", "accountname", "company", "business"},
    "domain": {"domainappend", "emaildomain"},
    "domainappend": {"domain", "emaildomain"},
    "emaildomain": {"domain", "domainappend"},
    "website": {"providerwebsite", "websiteurl", "url", "webaddress", "homepage"},
    "providerwebsite": {"website", "websiteurl", "url", "webaddress", "homepage"},
    "websiteurl": {"website", "providerwebsite", "url", "webaddress", "homepage"},
    "webaddress": {"website", "providerwebsite", "websiteurl", "url", "homepage"},
    "homepage": {"website", "providerwebsite", "websiteurl", "url"},
    "url": {"website", "providerwebsite", "websiteurl", "webaddress", "homepage"},
}


def _fallback_header_mappings(
    source_cols: List[str], ref_cols: List[str]
) -> List[Dict[str, Any]]:
    """Simple header-based fallback when AutoMapper produces no mappings."""

    normalized_ref: Dict[str, List[str]] = {}
    for col in ref_cols:
        norm = _normalize_header(col)
        if not norm:
            continue
        normalized_ref.setdefault(norm, []).append(col)

    used_ref: set[str] = set()
    fallback: List[Dict[str, Any]] = []

    for col in source_cols:
        norm = _normalize_header(col)
        if not norm:
            continue
        candidate_keys = [norm]
        candidate_keys.extend(sorted(HEADER_SYNONYMS.get(norm, ())))
        target = None
        for key in candidate_keys:
            for cand in normalized_ref.get(key, []):
                if cand not in used_ref:
                    target = cand
                    break
            if target:
                break
        if target is None and source_cols is ref_cols:
            target = col
        if target is None:
            continue
        used_ref.add(target)
        fallback.append(
            {
                "source": col,
                "reference": target,
                "weight": 0.5,
                "confidence": 0.5,
                "reasons": [{"kind": "fallback_header_match", "score": 0.5}],
            }
        )

    return fallback


OBJECT_SAMPLE_ROWS = 5000


def _wrap_intelligence(
    config: dict, mapping_warnings: list[dict] | None = None
) -> dict:
    config_hash = _stable_intelligence_hash(config)
    config["hash"] = config_hash
    return {
        "intelligence_config": config,
        "configuration": config,
        "config_hash": config_hash,
        "mapping_warnings": mapping_warnings or [],
    }


def _normalize_mappings(
    mappings: Optional[List[Dict[str, Any]]],
) -> List[Dict[str, Any]]:
    normalized: List[Dict[str, Any]] = []
    if not isinstance(mappings, list):
        return normalized
    for mapping in mappings:
        if not isinstance(mapping, dict):
            continue
        entry = dict(mapping)
        ref_value = entry.get("reference") or entry.get("ref")
        if ref_value:
            entry["reference"] = ref_value
            entry.setdefault("ref", ref_value)
        normalized.append(entry)
    return normalized


def calculate_mapping_confidence(mappings):
    """Calculate overall confidence score for mappings"""
    if not mappings:
        return 0.0
    # Average the weights/confidence scores
    confidences = [m.get("weight", 0.5) for m in mappings]
    return round(sum(confidences) / len(confidences), 3) if confidences else 0.0


def calculate_mapping_coverage(mappings, source_df, ref_df):
    """Calculate how much of the data is covered by mappings"""
    if not mappings or source_df is None:
        return 0.0

    # Calculate coverage for source columns
    source_mapped = len({m["source"] for m in mappings if "source" in m})
    source_total = len(source_df.columns) if source_df is not None else 0

    # Calculate coverage for reference columns
    if ref_df is not None:
        ref_mapped = len({m["ref"] for m in mappings if "ref" in m})
        ref_total = len(ref_df.columns)
        # Average coverage of both datasets
        source_coverage = source_mapped / source_total if source_total > 0 else 0
        ref_coverage = ref_mapped / ref_total if ref_total > 0 else 0
        return round((source_coverage + ref_coverage) / 2, 3)
    else:
        # Dedupe mode - just source coverage
        return round(source_mapped / source_total, 3) if source_total > 0 else 0.0


@router.post("/analyze")
async def analyze_data(
    request: AnalyzeRequest,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> Dict[str, Any]:
    """
    Comprehensive data analysis using all engine intelligence.
    Returns ID columns, mappings, blocking strategies, and algorithm recommendations.
    """
    # Load data from job
    source_df, ref_df = await load_job_data(request.job_id)

    if source_df is None:
        raise HTTPException(404, "Job data not found")

    # Sample for performance
    if len(source_df) > request.sample_size:
        source_sample = source_df.sample(n=request.sample_size, random_state=42)
    else:
        source_sample = source_df

    ref_sample = None
    if ref_df is not None:
        if len(ref_df) > request.sample_size:
            ref_sample = ref_df.sample(n=request.sample_size, random_state=42)
        else:
            ref_sample = ref_df

    # 1. ID Column Detection with confidence
    source_id_detection = ColumnDetector.detect_id_column(source_sample)
    ref_id_detection = None
    if ref_sample is not None:
        ref_id_detection = ColumnDetector.detect_id_column(ref_sample)

    # 2. Enhanced Column Profiling with statistics and UI hints
    source_profile = profile_dataframe(source_sample)
    ref_profile = profile_dataframe(ref_sample) if ref_sample is not None else None

    # Add detailed column statistics
    def enhance_column_stats(df, profile):
        """Add fill rate, distinct ratio, samples, and UI hints"""
        enhanced_cols = []
        for col in df.columns:
            col_data = df[col]
            stats = {
                "name": col,
                "dtype": str(col_data.dtype),
                "fill_rate": float(col_data.notna().mean()),
                "distinct_count": int(col_data.nunique()),
                "distinct_ratio": float(col_data.nunique() / len(col_data))
                if len(col_data) > 0
                else 0,
                "null_count": int(col_data.isna().sum()),
                "samples": [str(x) for x in col_data.dropna().head(5).tolist()]
                if not col_data.empty
                else [],
                "inferred_type": _get_semantic_type(col, col_data),
                "ui_hints": {
                    "icon": _get_column_icon(_get_semantic_type(col, col_data)),
                    "tooltip": _get_column_tooltip(col, col_data),
                    "color": _get_column_color(_get_semantic_type(col, col_data)),
                },
            }
            enhanced_cols.append(stats)
        profile["columns_detailed"] = enhanced_cols
        return profile

    source_profile = enhance_column_stats(source_sample, source_profile)
    if ref_profile:
        ref_profile = enhance_column_stats(ref_sample, ref_profile)

    stats_cache = ColumnStatsCache()
    mapping_warnings: List[Dict[str, Any]] = []

    try:
        # 3. Mapping Suggestions
        mappings: List[Dict[str, Any]] = []
        if ref_sample is not None:
            mapper = AutoMapper(stats_cache)
            mappings = mapper.suggest(source_sample, ref_sample, threshold=75)
            mapping_warnings = list(mapper.last_warnings or [])

            # Enhance with algorithm recommendations
            for mapping in mappings:
                source_col = mapping["source"]
                ref_col = mapping["ref"]

                # Detect semantic types
                source_semantic = _get_semantic_type(
                    source_col, source_sample[source_col]
                )
                ref_semantic = _get_semantic_type(ref_col, ref_sample[ref_col])

                # Recommend algorithm based on semantic type
                mapping["recommended_algo"] = recommend_algorithm(
                    source_semantic,
                    ref_semantic,
                    source_sample[source_col],
                    ref_sample[ref_col],
                )

        # 4. Blocking Strategy Analysis
        mode = BlockingMode.MATCH if ref_df is not None else BlockingMode.DEDUPE
        blocking_strategy = determine_optimal_blocking_strategy(
            source_sample,
            ref_sample,
            mappings,
            mode,
            "default",
            weight_profile="balanced",
            stats_cache=stats_cache,
        )

        # 5. Data Quality Assessment
        quality_metrics = assess_data_quality(source_sample, ref_sample)

        # 6. Performance Estimation
        performance_estimate = estimate_performance(
            len(source_df),
            len(ref_df) if ref_df is not None else len(source_df),
            blocking_strategy,
            mode,
        )

        # Generate correlation_id for validation
        correlation_id = str(uuid.uuid4())

        # Store correlation_id in Redis with 24hr TTL
        from .jobs import get_redis

        redis_conn = await get_redis()
        if redis_conn:
            await redis_conn.set(
                f"correlation:{correlation_id}",
                json.dumps(
                    {
                        "job_id": request.job_id,
                        "account_id": str(account_id),
                        "created_at": pd.Timestamp.now().isoformat(),
                        "type": "analysis",
                    }
                ),
                ex=86400,
            )  # 24 hour TTL

        coverage = calculate_mapping_coverage(mappings, source_sample, ref_sample)

        return {
            "analysis_complete": True,
            "correlation_id": correlation_id,
            "source": {
                "rows": len(source_df),
                "columns": len(source_df.columns),
                "id_column": {
                    "detected": source_id_detection.column_name
                    if source_id_detection
                    else None,
                    "confidence": source_id_detection.score
                    if source_id_detection
                    else 0,
                    "system": source_id_detection.detected_system
                    if source_id_detection
                    else None,
                },
                "profile": source_profile,
            },
            "reference": {
                "rows": len(ref_df) if ref_df is not None else 0,
                "columns": len(ref_df.columns) if ref_df is not None else 0,
                "id_column": {
                    "detected": ref_id_detection.column_name
                    if ref_id_detection
                    else None,
                    "confidence": ref_id_detection.score if ref_id_detection else 0,
                    "system": ref_id_detection.detected_system
                    if ref_id_detection
                    else None,
                },
                "profile": ref_profile,
            }
            if ref_df is not None
            else None,
            "mappings": {
                "suggested": mappings,
                "confidence": calculate_mapping_confidence(mappings),
                "coverage": coverage,
                "warnings": mapping_warnings,
            },
            "blocking": {
                "recommended_strategy": blocking_strategy,
                "estimated_reduction": blocking_strategy.get("raw_metrics", {}).get(
                    "RR", 0
                ),
                "quality_score": blocking_strategy.get("quality_score", 0),
            },
            "quality": quality_metrics,
            "performance_estimate": performance_estimate,
        }
    finally:
        stats_cache.clear()
        clear_domain_cache()


def assess_data_quality(source_df, ref_df=None):
    """
    Assess the quality of source and reference dataframes.
    Returns quality metrics for matching potential.
    """
    quality_metrics = {}

    # Source data quality
    if source_df is not None and not source_df.empty:
        source_size = source_df.size
        source_nulls = source_df.isnull().sum().sum()
        source_quality = 1.0 - (source_nulls / source_size) if source_size > 0 else 0

        # Additional quality indicators
        # Convert dict columns to strings for duplicate detection
        try:
            df_for_dup_check = source_df.copy()
            for col in df_for_dup_check.columns:
                if df_for_dup_check[col].dtype == "object":
                    # Check if column contains dicts
                    sample = (
                        df_for_dup_check[col].dropna().iloc[0]
                        if not df_for_dup_check[col].dropna().empty
                        else None
                    )
                    if isinstance(sample, dict):
                        df_for_dup_check[col] = df_for_dup_check[col].apply(
                            lambda x: str(x) if isinstance(x, dict) else x
                        )
            source_duplicates = df_for_dup_check.duplicated().sum()
        except:
            # If duplicate check fails, just set to 0
            source_duplicates = 0
        source_duplicate_ratio = (
            source_duplicates / len(source_df) if len(source_df) > 0 else 0
        )

        quality_metrics["source_quality"] = round(source_quality, 3)
        quality_metrics["source_completeness"] = round(source_quality, 3)
        quality_metrics["source_duplicate_ratio"] = round(source_duplicate_ratio, 3)
        quality_metrics["source_rows"] = len(source_df)
        quality_metrics["source_columns"] = len(source_df.columns)
    else:
        quality_metrics["source_quality"] = 0
        quality_metrics["source_completeness"] = 0
        quality_metrics["source_duplicate_ratio"] = 0
        quality_metrics["source_rows"] = 0
        quality_metrics["source_columns"] = 0

    # Reference data quality
    if ref_df is not None and not ref_df.empty:
        ref_size = ref_df.size
        ref_nulls = ref_df.isnull().sum().sum()
        ref_quality = 1.0 - (ref_nulls / ref_size) if ref_size > 0 else 0

        # Convert dict columns to strings for duplicate detection
        try:
            df_for_dup_check = ref_df.copy()
            for col in df_for_dup_check.columns:
                if df_for_dup_check[col].dtype == "object":
                    # Check if column contains dicts
                    sample = (
                        df_for_dup_check[col].dropna().iloc[0]
                        if not df_for_dup_check[col].dropna().empty
                        else None
                    )
                    if isinstance(sample, dict):
                        df_for_dup_check[col] = df_for_dup_check[col].apply(
                            lambda x: str(x) if isinstance(x, dict) else x
                        )
            ref_duplicates = df_for_dup_check.duplicated().sum()
        except:
            # If duplicate check fails, just set to 0
            ref_duplicates = 0
        ref_duplicate_ratio = ref_duplicates / len(ref_df) if len(ref_df) > 0 else 0

        quality_metrics["reference_quality"] = round(ref_quality, 3)
        quality_metrics["reference_completeness"] = round(ref_quality, 3)
        quality_metrics["reference_duplicate_ratio"] = round(ref_duplicate_ratio, 3)
        quality_metrics["reference_rows"] = len(ref_df)
        quality_metrics["reference_columns"] = len(ref_df.columns)
    else:
        quality_metrics["reference_quality"] = 0
        quality_metrics["reference_completeness"] = 0
        quality_metrics["reference_duplicate_ratio"] = 0
        quality_metrics["reference_rows"] = 0
        quality_metrics["reference_columns"] = 0

    # Calculate match potential based on data quality
    if ref_df is not None:
        # For matching mode
        avg_quality = (
            quality_metrics["source_quality"] + quality_metrics["reference_quality"]
        ) / 2
        size_ratio = (
            min(len(source_df), len(ref_df)) / max(len(source_df), len(ref_df))
            if max(len(source_df), len(ref_df)) > 0
            else 0
        )
        quality_metrics["match_potential"] = round(
            avg_quality * 0.7 + size_ratio * 0.3, 3
        )
    else:
        # For dedupe mode
        quality_metrics["match_potential"] = round(
            quality_metrics["source_quality"]
            * (1 - quality_metrics["source_duplicate_ratio"]),
            3,
        )

    # Overall quality score
    quality_metrics["overall_score"] = round(
        quality_metrics.get("match_potential", 0) * 0.5
        + quality_metrics.get("source_quality", 0) * 0.3
        + quality_metrics.get("reference_quality", 0) * 0.2,
        3,
    )

    return quality_metrics


@router.post("/suggest-mappings")
async def suggest_column_mappings(
    request: MappingSuggestionRequest,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> Dict[str, Any]:
    """
    Get intelligent column mapping suggestions with algorithm recommendations.
    """
    source_df, ref_df = await load_job_data(request.job_id)

    if source_df is None or ref_df is None:
        raise HTTPException(400, "Both source and reference data required")

    # Use AutoMapper for intelligent suggestions
    mapper = AutoMapper()
    mappings = mapper.suggest(source_df, ref_df, request.threshold)

    # Enhance with multi-algorithm recommendations if enabled
    if request.enable_multi_algo:
        for mapping in mappings:
            profile = create_column_profile(
                source_df[mapping["source"]], ref_df[mapping["ref"]]
            )

            # Test different algorithms
            algo_scores = test_algorithms_on_sample(
                source_df[mapping["source"]], ref_df[mapping["ref"]], profile
            )

            # Select best algorithm
            best_algo = max(algo_scores.items(), key=lambda x: x[1])
            mapping["preferred_algo"] = best_algo[0]
            mapping["algo_confidence"] = best_algo[1]

    return {
        "mappings": mappings,
        "total_mapped": len(mappings),
        "unmapped_source": [
            col
            for col in source_df.columns
            if col not in [m["source"] for m in mappings]
        ],
        "unmapped_reference": [
            col for col in ref_df.columns if col not in [m["ref"] for m in mappings]
        ],
        "multi_algo_enabled": request.enable_multi_algo,
    }


@router.post("/optimize-blocking")
async def optimize_blocking_strategy(
    request: BlockingAnalysisRequest,
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> Dict[str, Any]:
    """
    Determine optimal blocking strategy with custom weight profiles.
    """
    source_df, ref_df = await load_job_data(request.job_id)

    if source_df is None:
        raise HTTPException(404, "Job data not found")

    # Validate custom weights if provided
    if request.custom_weights:
        is_valid, msg = BlockingWeightProfile.validate_weights(request.custom_weights)
        if not is_valid:
            raise HTTPException(400, f"Invalid custom weights: {msg}")

    # Determine mode
    mode = BlockingMode.MATCH if ref_df is not None else BlockingMode.DEDUPE

    # Run blocking analysis with progress tracking
    progress_data = []

    def progress_callback(update):
        progress_data.append(update)

    stats_cache = ColumnStatsCache()

    try:
        # Determine optimal strategy
        strategy = determine_optimal_blocking_strategy(
            source_df,
            ref_df,
            mappings=[],  # Will use all columns
            mode=mode,
            ruleset="default",
            weight_profile=request.weight_profile,
            custom_weights=request.custom_weights,
            progress_callback=progress_callback,
            stats_cache=stats_cache,
        )

        # Validate candidates
        if strategy.get("apply_blocking"):
            candidates = validate_blocking_candidates(
                [
                    (
                        strategy["source_block_col"],
                        strategy.get("ref_block_col", strategy["source_block_col"]),
                    )
                ],
                source_df,
                ref_df,
                mode,
            )
            strategy["validation"] = candidates[0] if candidates else {}

        # Detect optimal preprocessing variants
        if strategy.get("source_block_col"):
            variants = detect_optimal_variants(
                source_df[strategy["source_block_col"]],
                strategy["source_block_col"],
                strategy.get("validation", {}).get("src_metrics", {}),
            )
            strategy["preprocessing_variants"] = variants

        def get_alternative_strategies(progress_data):
            """Extract alternative blocking strategies from progress data"""
            if not progress_data:
                return []

            # Sort strategies by quality score and return top alternatives
            alternatives = []
            for data in progress_data[:5]:  # Top 5 alternatives
                if isinstance(data, dict) and "quality_score" in data:
                    alternatives.append(
                        {
                            "column": data.get("column", "unknown"),
                            "variant": data.get("variant", "default"),
                            "quality_score": data.get("quality_score", 0),
                            "reduction_rate": data.get("raw_metrics", {}).get("RR", 0),
                        }
                    )

            # Sort by quality score descending
            alternatives.sort(key=lambda x: x.get("quality_score", 0), reverse=True)
            return alternatives[1:4]  # Return 2nd to 4th best (1st is the optimal)

        # Calculate impact metrics
        n_src = len(source_df)
        n_ref = len(ref_df) if ref_df is not None else n_src
        pairs_before = n_src * n_ref

        # Calculate blocks created and pairs after if blocking is applied
        blocks_created = 0
        pairs_after = pairs_before
        avg_block_size = 0
        max_block_size = 0

        if strategy.get("apply_blocking") and strategy.get("source_block_col"):
            # Create block keys for analysis
            src_col = strategy["source_block_col"]
            key_len = strategy.get("block_key_len", 3)

            # Apply preprocessing to get block keys
            src_block_keys = source_df[src_col].fillna("").astype(str).str[:key_len]
            block_counts = src_block_keys.value_counts(dropna=False)
            blocks_created = int(block_counts.shape[0])
            avg_block_size = float(block_counts.mean() or 0)
            max_block_size = int(block_counts.max() or 0)

            # Estimate pairs after blocking
            if ref_df is not None:
                ref_col = strategy.get("ref_block_col", src_col)
                ref_block_keys = ref_df[ref_col].fillna("").astype(str).str[:key_len]
                ref_block_counts = ref_block_keys.value_counts(dropna=False)

                # Join on block keys to estimate pairs
                joined = (
                    block_counts.to_frame("src")
                    .join(ref_block_counts.to_frame("ref"), how="inner")
                    .fillna(0)
                )
                pairs_after = int((joined["src"] * joined["ref"]).sum())
            else:
                # Dedupe mode - pairs within blocks
                pairs_after = int((block_counts * (block_counts - 1) / 2).sum())

        pairs_reduction_pct = (
            0 if pairs_before == 0 else 100.0 * (1 - (pairs_after / pairs_before))
        )

        # Build impact metrics
        impact = {
            "blocks_created": blocks_created,
            "avg_block_size": avg_block_size,
            "max_block_size": max_block_size,
            "pairs_before": int(pairs_before),
            "pairs_after": int(pairs_after),
            "pairs_reduction_pct": pairs_reduction_pct,
            "est_time_saved_ms": int(
                max(pairs_before - pairs_after, 0) / 5000
            ),  # Rough estimate
            "memory_mb": round(
                float(blocks_created * 0.01), 2
            ),  # Placeholder calculation
        }

        # Add distribution histogram if blocks were created
        distribution = {}
        if blocks_created > 0 and "block_counts" in locals():
            distribution = {
                "histogram": block_counts.head(50).to_dict(),
                "total_blocks": blocks_created,
            }

        return {
            "optimal_strategy": strategy,
            "alternatives": get_alternative_strategies(progress_data),
            "weight_profile_used": request.weight_profile,
            "custom_weights": request.custom_weights,
            "analysis_summary": {
                "strategies_evaluated": len(progress_data),
                "best_score": strategy.get("quality_score", 0),
                "estimated_reduction": strategy.get("raw_metrics", {}).get("RR", 0),
            },
            "impact": impact,
            "distribution": distribution,
        }
    finally:
        stats_cache.clear()
        clear_domain_cache()


@router.post("/auto-configure")
async def auto_configure_job(
    job_id: str = Body(None),
    optimization_goal: str = Body(
        "balanced", description="balanced, speed, or accuracy"
    ),
    goal: str = Body(None),  # For schema-based requests
    source: Dict[str, Any] = Body(None),  # Schema info
    reference: Dict[str, Any] = Body(None),  # Schema info
    source_object_id: Optional[str] = Body(None),
    reference_object_id: Optional[str] = Body(None),
    constraints: Dict[str, Any] = Body(None),  # Additional constraints
    mapping_overrides: Optional[List[Dict[str, Any]]] = Body(None),
    db: AsyncSession = Depends(get_session),
    account_id: UUID = Depends(get_current_account),
) -> JSONResponse:
    """
    Generate complete optimal configuration for a job.
    Accepts either:
    1. job_id - loads data from Redis
    2. source/reference schemas - generates config from schema only
    """
    trace_id = uuid.uuid4().hex.replace("-", "")
    req_log = LoggerAdapter(log, {"trace_id": trace_id})

    t0 = time.perf_counter()

    def ms() -> int:
        return int((time.perf_counter() - t0) * 1000)

    req_log.info(
        "auto-config[%s]: start (account=%s job=%s source_schema=%s reference_schema=%s source_object=%s reference_object=%s)",
        trace_id,
        str(account_id),
        job_id or "-",
        bool(source),
        bool(reference),
        source_object_id or "-",
        reference_object_id or "-",
    )

    request_payload = constraints if isinstance(constraints, dict) else {}
    normalized_threshold = None
    thr = None
    if isinstance(request_payload, dict):
        thr = request_payload.get("threshold")
    if thr is not None:
        try:
            thr = float(thr)
            if 1.0 < thr <= 100.0:
                thr = thr / 100.0
            thr = max(0.0, min(thr, 1.0))
        except Exception:
            thr = None
    if thr is not None:
        normalized_threshold = thr
        request_payload["threshold"] = thr

    # Route time budget metadata for clients
    route_timeout_s = 30
    raw_budget = os.getenv("B2B_BUDGET_S")
    try:
        b2b_budget_s = float(raw_budget) if raw_budget is not None else 2.0
    except Exception:
        req_log.warning("Invalid B2B_BUDGET_S=%s; defaulting to 2.0", raw_budget)
        b2b_budget_s = 2.0
    # Tiny budget for B2B anchor/mapping analysis before fallback
    b2b_budget_s = max(0.1, min(b2b_budget_s, 10.0))

    def _columns_payload_from(src_df, ref_df, src_schema=None, ref_schema=None):
        src = list(src_df.columns) if src_df is not None else (src_schema or [])
        ref = list(ref_df.columns) if ref_df is not None else (ref_schema or [])
        return {"source": [str(c) for c in src], "reference": [str(c) for c in ref]}

    def _envelope(config: dict, warnings, columns: dict) -> dict:
        cfg = config or {}
        warnings_list = list(warnings or [])
        columns_dict = columns if isinstance(columns, dict) else {}
        raw_source = columns_dict.get("source") or []
        raw_reference = columns_dict.get("reference") or []
        safe_source = [str(col) for col in raw_source if col is not None]
        safe_reference = [str(col) for col in raw_reference if col is not None]
        safe_columns = {"source": safe_source, "reference": safe_reference}

        # Extract mappings from config before wrapping
        mappings_source = cfg.get("mappings") or []
        clean_mappings: List[Dict[str, Any]] = []
        if isinstance(mappings_source, list):
            for item in mappings_source:
                if isinstance(item, dict):
                    clean_mappings.append(dict(item))

        # Ensure mappings are in config before wrapping
        cfg["mappings"] = clean_mappings
        cfg.setdefault("field_mappings", clean_mappings)

        wrapped = _wrap_intelligence(cfg, warnings_list)
        data_payload = wrapped.get("intelligence_config")
        if not isinstance(data_payload, dict):
            data_payload = dict(cfg) if isinstance(cfg, dict) else {}
        else:
            data_payload = dict(data_payload)

        # Ensure mappings are preserved in data_payload
        data_payload["mappings"] = clean_mappings
        data_payload.setdefault("field_mappings", clean_mappings)
        data_payload["requires_confirmation"] = bool(
            data_payload.get("requires_confirmation", False)
        )

        # Ensure hash is included in data_payload
        if "hash" in cfg or "config_hash" in wrapped:
            data_payload["hash"] = cfg.get("hash") or wrapped.get("config_hash")
            data_payload["config_hash"] = cfg.get("hash") or wrapped.get("config_hash")

        resp: Dict[str, Any] = {
            "success": True,
            "code": 200,
            "error": None,
            "data": data_payload,
            "config": data_payload,
            "warnings": warnings_list,
            "mappings": clean_mappings,
            "columns": safe_columns,
            "sourceColumns": safe_source,
            "referenceColumns": safe_reference,
        }
        resp.update(wrapped)
        resp.setdefault("intelligence_config", data_payload)
        resp.setdefault("configuration", data_payload)

        meta = dict(resp.get("meta") or {})
        meta.update(
            {
                "schema_version": "v1",
                "route_timeout_s": route_timeout_s,
                "trace_id": trace_id,
            }
        )
        resp["meta"] = meta

        req_log.info(
            "auto-config[%s]: returning envelope (requires_confirmation=%s, mappings=%d, cols src=%d ref=%d)",
            trace_id,
            data_payload["requires_confirmation"],
            len(clean_mappings),
            len(safe_source),
            len(safe_reference),
        )

        # Debug logging to see actual structure
        import json

        req_log.info(
            "auto-config[%s]: intelligence_config has mappings=%d, data has mappings=%d, root has mappings=%d",
            trace_id,
            len(resp.get("intelligence_config", {}).get("mappings", [])),
            len(resp.get("data", {}).get("mappings", [])),
            len(resp.get("mappings", [])),
        )

        # Log first 500 chars of response for debugging
        resp_str = json.dumps(resp, default=str)[:500]
        req_log.info("auto-config[%s]: response preview: %s", trace_id, resp_str)

        return resp

    def _respond(config: dict, warnings, columns: dict) -> JSONResponse:
        envelope = _envelope(config, warnings, columns)
        return JSONResponse(
            content=envelope,
            status_code=200,
            headers={
                "X-FM-Trace-Id": trace_id,
                "X-FM-Route-Timeout": str(route_timeout_s),
            },
        )

    # Handle schema-based request (from IntelligenceClient and Google Sheets)
    if source is not None and job_id is None:
        # Use the goal parameter if provided, otherwise optimization_goal
        if goal:
            optimization_goal = "balanced"  # Default for L2A

        # Generate config based on schemas only
        source_schema = source.get("schema", [])
        ref_schema = reference.get("schema", []) if reference else []

        # For dedupe, skip system fields that will always differ
        is_dedupe = goal == "dedupe" or not reference or ref_schema == source_schema
        skip_fields = []
        if is_dedupe:
            skip_fields = [
                "id",
                "createddate",
                "lastmodifieddate",
                "isdeleted",
                "attributes",
                "systemmodstamp",
                "createdbyid",
                "lastmodifiedbyid",
                "ownerid",
            ]

        # Create improved field mappings with smarter logic
        field_mappings = []
        used_ref_cols = set()

        # Normalize column names for better matching
        def normalize_for_matching(col_name):
            """Normalize column name for matching purposes"""
            normalized = col_name.lower()
            # Remove common prefixes/suffixes
            for prefix in ["dhc_", "dhc ", "zoominfo_", "zoominfo "]:
                if normalized.startswith(prefix):
                    normalized = normalized[len(prefix) :]
            for suffix in ["_append", " append", "_id", " id"]:
                if normalized.endswith(suffix):
                    normalized = normalized[: -len(suffix)]
            # Normalize common terms
            normalized = normalized.replace("_", " ").replace("-", " ")
            return normalized.strip()

        # First pass: exact normalized matches
        for src_col in source_schema:
            if is_dedupe and src_col.lower() in skip_fields:
                continue

            src_norm = normalize_for_matching(src_col)

            for ref_col in ref_schema:
                if ref_col in used_ref_cols:
                    continue
                if is_dedupe and ref_col.lower() in skip_fields:
                    continue

                ref_norm = normalize_for_matching(ref_col)

                # Strong match on normalized names
                if src_norm == ref_norm:
                    algo = _get_algo_for_field(src_col)
                    field_mappings.append(
                        {
                            "source": src_col,
                            "reference": ref_col,
                            "algorithm": algo,
                            "weight": 0.9,
                        }
                    )
                    used_ref_cols.add(ref_col)
                    break

        # Second pass: semantic matching for common patterns
        semantic_mappings = [
            # Company/Account mappings
            (
                ["company", "account", "organization", "org"],
                ["company", "account", "organization", "org", "name"],
                "token_set_ratio",
                0.8,
            ),
            # Domain mappings - be more specific
            (["domain", "email domain"], ["domain"], "domain", 0.9),
            # Website mappings - keep separate from domain
            (
                ["website", "web", "url"],
                ["website", "web", "url"],
                "token_set_ratio",
                0.7,
            ),
            # Location mappings
            (["city"], ["city"], "token_set_ratio", 0.8),
            (["state", "province"], ["state", "province"], "token_set_ratio", 0.8),
            # Email mappings
            (["email"], ["email"], "exact", 0.8),
            # Phone mappings
            (["phone", "tel"], ["phone", "tel"], "exact", 0.7),
        ]

        for src_col in source_schema:
            if any(m["source"] == src_col for m in field_mappings):
                continue  # Already mapped

            src_lower = src_col.lower()

            for src_terms, ref_terms, algo, weight in semantic_mappings:
                if not any(term in src_lower for term in src_terms):
                    continue

                for ref_col in ref_schema:
                    if ref_col in used_ref_cols:
                        continue

                    ref_lower = ref_col.lower()

                    if any(term in ref_lower for term in ref_terms):
                        # Additional validation for specific cases
                        # Don't map URLs to domains or websites
                        if "url" in src_lower and "profile" in src_lower:
                            continue  # Skip profile URLs
                        if "division" in src_lower or "division" in ref_lower:
                            continue  # Skip division fields
                        if ("id" in ref_lower and "id" not in src_lower) or (
                            "id" in src_lower and "id" not in ref_lower
                        ):
                            continue  # Don't cross-map ID fields with non-ID fields

                        field_mappings.append(
                            {
                                "source": src_col,
                                "reference": ref_col,
                                "algorithm": algo,
                                "weight": weight,
                            }
                        )
                        used_ref_cols.add(ref_col)
                        break

                if any(m["source"] == src_col for m in field_mappings):
                    break

        # If no mappings found, create minimal default mappings
        if not field_mappings and len(source_schema) > 0 and len(ref_schema) > 0:
            # Only map truly similar columns as fallback
            for src_col in source_schema[:2]:  # Limit fallback mappings
                for ref_col in ref_schema[:2]:
                    if ref_col not in used_ref_cols:
                        src_type = _get_semantic_type(src_col, pd.Series([]))
                        ref_type = _get_semantic_type(ref_col, pd.Series([]))
                        if src_type == ref_type and src_type != "text":
                            field_mappings.append(
                                {
                                    "source": src_col,
                                    "reference": ref_col,
                                    "algorithm": "token_set_ratio",
                                    "weight": 0.4,
                                }
                            )
                            used_ref_cols.add(ref_col)
                            break

        # Build response compatible with Google Sheets
        # The Sheets code expects intelligence_config.mappings
        config = {
            "profile": goal or optimization_goal or "balanced",
            "mappings": field_mappings,  # Google Sheets expects "mappings"
            "threshold": normalized_threshold
            if normalized_threshold is not None
            else (0.80 if goal == "match" else 0.85),
            "algorithm": "WRatio",
            "blocking_strategy": {
                "apply_blocking": len(source_schema) > 5 or len(ref_schema) > 5,
                "strategy": "adaptive",
            },
            "options": {
                "explain": True,
                "has_header": True,
                "reference_has_header": True if ref_schema else False,
            },
        }
        if normalized_threshold is not None:
            config["threshold"] = normalized_threshold

        config["mappings"] = _normalize_mappings(config.get("mappings"))
        config["field_mappings"] = config["mappings"]

        # Wrap in intelligence_config for Google Sheets compatibility
        columns = _columns_payload_from(None, None, source_schema, ref_schema)
        clear_domain_cache()
        return _respond(config, [], columns)

    # Original job/object based logic
    source_df: pd.DataFrame | None = None
    ref_df: pd.DataFrame | None = None

    if job_id:
        source_df, ref_df = await load_job_data(job_id)
    elif source_object_id:
        source_df, ref_df = await load_object_data(
            source_object_id, reference_object_id
        )
    else:
        raise HTTPException(
            400, "Either job_id, source schema, or source_object_id must be provided"
        )

    if source_df is None:
        raise HTTPException(404, "Source data not found")

    job_ref = job_id or source_object_id

    # Run comprehensive analysis
    config = {}
    columns = _columns_payload_from(source_df, ref_df)
    deadline = time.perf_counter() + b2b_budget_s

    req_log.info("auto-config[%s]: pre-b2b t=%dms", trace_id, ms())

    # NEW: Segment fingerprinting for bandits
    def fingerprint_segment(source_df, ref_df=None):
        """Create segment identifier based on data characteristics"""
        segments = []

        # Size category
        size = len(source_df)
        if size < 1000:
            segments.append("small")
        elif size < 10000:
            segments.append("medium")
        else:
            segments.append("large")

        # Mode
        segments.append("dedupe" if ref_df is None else "match")

        # Data quality
        null_ratio = source_df.isnull().sum().sum() / source_df.size
        if null_ratio < 0.1:
            segments.append("high_quality")
        elif null_ratio < 0.3:
            segments.append("medium_quality")
        else:
            segments.append("low_quality")

        # Check for domain columns
        if any("domain" in col.lower() for col in source_df.columns):
            segments.append("has_domain")

        return "_".join(segments)

    # Create segment identifier
    segment = fingerprint_segment(source_df, ref_df)
    mode = "dedupe" if ref_df is None else "match"

    # NEW: Initialize bandits with Redis connection
    from .jobs import get_redis

    redis_conn = await get_redis()
    threshold_bandit = ThresholdBandit(redis_conn) if redis_conn else None
    blocking_bandit = BlockingBandit(redis_conn) if redis_conn else None

    # 1. ID columns
    source_id = auto_select_id_column(source_df)
    config["source_id_column"] = source_id

    if ref_df is not None:
        ref_id = auto_select_id_column(ref_df)
        config["reference_id_column"] = ref_id

    # 2. Mappings
    mapping_warnings: List[Dict[str, Any]] = []
    stats_cache = ColumnStatsCache()  # Initialize stats cache for performance

    if ref_df is not None:
        if time.perf_counter() >= deadline:
            fallback_cfg = {
                "profile": goal or optimization_goal or "balanced",
                "mappings": [],
                "threshold": normalized_threshold
                if normalized_threshold is not None
                else 0.80,
                "requires_confirmation": True,
                "confirmationMessage": "Analysis exceeded time budget. Please confirm anchors (Company/Domain or Email+Company).",
            }
            req_log.info(
                "auto-config[%s]: B2B timeout prior to mappings t=%dms", trace_id, ms()
            )
            return _respond(fallback_cfg, ["B2B timeout"], columns)

        mapper = AutoMapper(stats_cache)
        mappings = mapper.suggest(source_df, ref_df)
        mapping_warnings = list(mapper.last_warnings or [])

        config["mappings"] = mappings
    else:
        # Dedupe mode - identity mappings
        config["mappings"] = [
            {"source": col, "reference": col, "weight": 1.0}
            for col in source_df.columns
            if col != source_id
        ]
        mapping_warnings = []

    if mapping_overrides and ref_df is not None:
        override_candidates: list[Dict[str, Any]] = []
        for item in mapping_overrides:
            if not isinstance(item, dict):
                continue
            src_col = item.get("source")
            ref_col = item.get("reference") or item.get("ref")
            if not src_col or not ref_col:
                continue
            if src_col not in source_df.columns or ref_col not in ref_df.columns:
                continue
            override_candidates.append({"source": src_col, "reference": ref_col})

        if override_candidates:
            enriched, override_warnings = engine_calculate_mapping_weights(
                override_candidates,
                source_df,
                ref_df,
                stats_cache=stats_cache,
            )
            processed_overrides = enriched if enriched else override_candidates
            manual_overrides: list[Dict[str, Any]] = []
            for entry in processed_overrides:
                normalized = dict(entry)
                src = normalized.get("source")
                ref = normalized.get("reference") or normalized.get("ref")
                if not src or not ref:
                    continue
                normalized["source"] = src
                normalized["reference"] = ref
                normalized["ref"] = ref
                normalized["role"] = "identity"
                normalized["weight"] = 1.0
                normalized["confidence"] = 1.0
                normalized.setdefault("locked", True)
                manual_overrides.append(normalized)

            if manual_overrides:
                override_keys = {
                    (m.get("source"), m.get("reference")) for m in manual_overrides
                }
                existing_mappings = config.get("mappings") or []
                filtered_existing: list[Dict[str, Any]] = []
                for mapping in existing_mappings:
                    src = mapping.get("source")
                    ref = mapping.get("reference") or mapping.get("ref")
                    if (src, ref) in override_keys:
                        continue
                    filtered_existing.append(mapping)

                config["mappings"] = manual_overrides + filtered_existing
                mapping_warnings.extend(override_warnings or [])
                mapping_warnings.append(
                    {
                        "type": "manual_override",
                        "message": f"Using {len(manual_overrides)} manual mapping overrides",
                    }
                )

    # Fallback: header intersection when no mappings resolved (common for new tenants)
    if not config.get("mappings"):
        fallback = _fallback_header_mappings(
            list(source_df.columns),
            list(ref_df.columns) if ref_df is not None else list(source_df.columns),
        )
        if fallback:
            config["mappings"] = fallback
            mapping_warnings.append(
                {
                    "reason": "fallback_header_match",
                    "details": "AutoMapper produced no mappings; using header intersection fallback.",
                }
            )
            mappings = fallback

    if ref_df is not None:
        for mapping in config.get("mappings", []):
            ref_col = mapping.get("reference") or mapping.get("ref")
            if not ref_col or ref_col not in ref_df.columns:
                continue
            var_profile = create_column_profile(
                source_df[mapping["source"]], ref_df[ref_col]
            )
            if optimization_goal == "speed":
                mapping["preferred_algo"] = "WRatio"
            elif optimization_goal == "accuracy":
                mapping["preferred_algo"] = select_accurate_algorithm(var_profile)
            else:
                mapping["preferred_algo"] = select_balanced_algorithm(var_profile)

    req_log.info(
        "auto-config[%s]: post-b2b t=%dms (mappings=%d requires_confirmation=%s)",
        trace_id,
        ms(),
        len(config.get("mappings") or []),
        bool(config.get("requires_confirmation", False)),
    )

    # 3. Blocking strategy
    blocking_strategy_name = "none"
    blocking_exploration = False

    if time.perf_counter() >= deadline:
        fallback_cfg = {
            "profile": goal or optimization_goal or "balanced",
            "mappings": config.get("mappings", []),
            "threshold": normalized_threshold
            if normalized_threshold is not None
            else 0.80,
            "requires_confirmation": True,
            "confirmationMessage": "Analysis exceeded time budget. Please confirm anchors (Company/Domain or Email+Company).",
        }
        req_log.info(
            "auto-config[%s]: B2B timeout prior to blocking t=%dms", trace_id, ms()
        )
        return _respond(fallback_cfg, ["B2B timeout"], columns)

    # NEW: Select blocking using bandit if available
    if blocking_bandit:
        available_columns = list(source_df.columns)
        blocking_selection = await blocking_bandit.select(segment, available_columns)

        if blocking_selection.get("apply_blocking"):
            blocking = {
                "apply_blocking": True,
                "source_block_col": blocking_selection.get("column"),
                "ref_block_col": blocking_selection.get("column"),
                "block_key_len": blocking_selection.get("key_length", 3),
                "variant": blocking_selection.get("variant", "alnum"),
            }
            blocking_strategy_name = blocking_selection.get("strategy")
            blocking_exploration = blocking_selection.get("exploration", False)
        else:
            blocking = {"apply_blocking": False}
            blocking_strategy_name = "none"
    else:
        # Fallback to original logic
        weight_profile = {
            "speed": "high_recall",
            "accuracy": "high_precision",
            "balanced": "balanced",
        }[optimization_goal]

        mode_enum = BlockingMode.MATCH if ref_df is not None else BlockingMode.DEDUPE
        blocking = determine_optimal_blocking_strategy(
            source_df,
            ref_df,
            config["mappings"],
            mode_enum,
            "default",
            weight_profile=weight_profile,
            stats_cache=stats_cache,
        )
        blocking_strategy_name = blocking.get("strategy", "adaptive")

    config.update(
        {
            "blocking_strategy": blocking,
            "apply_blocking": blocking.get("apply_blocking", True),
            "block_key_length": blocking.get("block_key_len", 3),
            "preprocessing_variant": blocking.get(
                "variant", blocking.get("preproc", "alnum")
            ),
        }
    )

    req_log.info("auto-config[%s]: post-blocking t=%dms", trace_id, ms())

    config["mapping_warnings"] = mapping_warnings

    # 4. Algorithm and threshold
    # NEW: Select threshold using bandit if available
    threshold_policy = None
    threshold_exploration = False

    if threshold_bandit:
        threshold_selection = await threshold_bandit.select(segment)
        sel_threshold = threshold_selection.get("threshold")
        try:
            sel_threshold = float(sel_threshold)
        except (TypeError, ValueError):
            sel_threshold = None
        if sel_threshold is not None and 1.0 < sel_threshold <= 100.0:
            sel_threshold = sel_threshold / 100.0
        if sel_threshold is not None:
            sel_threshold = max(0.0, min(sel_threshold, 1.0))
            config["threshold"] = sel_threshold
        threshold_policy = threshold_selection.get("policy")
        threshold_exploration = threshold_selection.get("exploration", False)

        # Log if we're exploring
        if threshold_exploration:
            req_log.info(f"Exploring threshold policy: {threshold_policy}")
    else:
        # Fallback to original logic
        if optimization_goal == "speed":
            config["algorithm"] = "WRatio"
            config["threshold"] = 0.75
        elif optimization_goal == "accuracy":
            config["algorithm"] = "JaroWinkler"
            config["threshold"] = 0.85
        else:
            config["algorithm"] = "WRatio"
            config["threshold"] = 0.80

    if normalized_threshold is not None:
        config["threshold"] = normalized_threshold

    # 5. Match mode - best match only vs all matches above threshold
    # For most use cases, best_match_only should be True to avoid duplicates
    config["best_match_only"] = (
        True  # Return only the best match for each source record
    )
    config["max_matches_per_source"] = 1  # Maximum matches per source record

    # 5. Advanced settings
    config["enable_multi_algo"] = optimization_goal == "accuracy"
    config["coverage_gate"] = 0.7 if optimization_goal == "accuracy" else 0.6
    config["max_workers"] = 4 if optimization_goal == "speed" else 2

    # 6. Performance estimate
    estimated_time = estimate_processing_time(
        len(source_df),
        len(ref_df) if ref_df is not None else len(source_df),
        blocking,
        config,
    )

    def _json_default(obj: Any) -> Any:
        if isinstance(obj, pd.Timestamp):
            return obj.isoformat()
        if isinstance(obj, np.generic):
            try:
                return obj.item()
            except Exception:
                return str(obj)
        if hasattr(obj, "isoformat"):
            try:
                return obj.isoformat()
            except Exception:
                pass
        if isinstance(obj, (set, tuple)):
            return list(obj)
        return str(obj)

    config["mappings"] = _normalize_mappings(config.get("mappings"))
    config["field_mappings"] = config["mappings"]
    config.setdefault("profile", goal or optimization_goal or "balanced")

    config_hash = _stable_intelligence_hash(config)
    config["hash"] = config_hash

    # Generate correlation_id for validation
    correlation_id = str(uuid.uuid4())

    # Store correlation_id in Redis with 24hr TTL
    if redis_conn:
        await redis_conn.set(
            f"correlation:{correlation_id}",
            json.dumps(
                {
                    "job_id": job_ref if job_ref else None,
                    "account_id": str(account_id),
                    "created_at": pd.Timestamp.now().isoformat(),
                    "config_hash": config_hash,
                }
            ),
            ex=86400,
        )  # 24 hour TTL

    # NEW: Log the decision for learning
    if redis_conn:
        decision_logger = _get_decision_logger(redis_conn)
        await decision_logger.log_decision(
            job_id=job_ref if job_ref else "unknown",
            tenant_id=str(account_id),
            segment=segment,
            mode=mode,
            blocking_choice=blocking_strategy_name,
            threshold_policy=threshold_policy if threshold_policy else "default",
            config_hash=config_hash,
        )

    # Build response with learning metadata if bandits are active
    validation_warnings = validate_configuration(config, source_df, ref_df)

    envelope = _envelope(config, mapping_warnings, columns)
    envelope.update(
        {
            "correlation_id": correlation_id,
            "optimization_goal": optimization_goal,
            "estimated_processing_time_seconds": estimated_time,
            "confidence_score": calculate_config_confidence(config, source_df, ref_df),
            "warnings": validation_warnings,
        }
    )

    # Add learning metadata if bandits are active
    if threshold_bandit or blocking_bandit:
        envelope["learning_metadata"] = {
            "segment": segment,
            "threshold_policy": threshold_policy if threshold_policy else "default",
            "threshold_exploration": threshold_exploration,
            "blocking_strategy": blocking_strategy_name,
            "blocking_exploration": blocking_exploration,
        }

    stats_cache.clear()
    clear_domain_cache()
    return JSONResponse(
        content=envelope,
        status_code=200,
        headers={
            "X-FM-Trace-Id": trace_id,
            "X-FM-Route-Timeout": str(route_timeout_s),
        },
    )


# Helper functions
async def load_job_data(job_id: str) -> tuple[pd.DataFrame | None, pd.DataFrame | None]:
    """
    Load source and reference CSVs for a job from Redis and parse with encoding detection.
    Reuses the read_csv_with_encoding helper from v2/jobs.py to avoid UTF-8 assumptions.
    """
    from .jobs import get_redis, read_csv_with_encoding  # reuse helpers

    redis_conn = await get_redis()
    if not redis_conn:
        return None, None

    metadata_raw = await redis_conn.get(f"job:{job_id}")
    if not metadata_raw:
        return None, None

    # Load raw CSV bytes
    source_bytes = await redis_conn.get(f"job:{job_id}:source_data")
    ref_bytes = await redis_conn.get(f"job:{job_id}:ref_data")

    source_df = read_csv_with_encoding(source_bytes) if source_bytes else None
    ref_df = read_csv_with_encoding(ref_bytes) if ref_bytes else None

    return source_df, ref_df


async def load_object_data(
    source_object_id: str,
    reference_object_id: Optional[str] = None,
    sample_rows: int = OBJECT_SAMPLE_ROWS,
) -> tuple[pd.DataFrame | None, pd.DataFrame | None]:
    """Load source/reference data from uploaded object_ids via the storage backend."""
    if not source_object_id:
        return None, None

    storage = get_storage_backend()

    try:
        source_df = await storage.read_sample(source_object_id, max_rows=sample_rows)
    except FileNotFoundError:
        raise HTTPException(404, f"source_object_id '{source_object_id}' not found")
    except ValueError as exc:
        raise HTTPException(400, str(exc))
    except Exception as exc:
        raise HTTPException(
            500, f"Failed to load source_object_id '{source_object_id}': {exc}"
        )

    ref_df: pd.DataFrame | None = None
    if reference_object_id:
        try:
            ref_df = await storage.read_sample(
                reference_object_id, max_rows=sample_rows
            )
        except FileNotFoundError:
            raise HTTPException(
                404, f"reference_object_id '{reference_object_id}' not found"
            )
        except ValueError as exc:
            raise HTTPException(400, str(exc))
        except Exception as exc:
            raise HTTPException(
                500,
                f"Failed to load reference_object_id '{reference_object_id}': {exc}",
            )

    return source_df, ref_df


def profile_dataframe(df: pd.DataFrame) -> Dict[str, Any]:
    """Generate comprehensive DataFrame profile."""
    if df is None:
        return {}

    # Convert dtypes to strings for JSON serialization
    dtype_counts = df.dtypes.value_counts()
    dtype_dict = {str(k): int(v) for k, v in dtype_counts.items()}

    profile = {
        "shape": list(df.shape),  # Convert tuple to list
        "memory_usage_mb": float(df.memory_usage(deep=True).sum() / 1024 / 1024),
        "dtypes": dtype_dict,
        "columns": [],
    }

    for col in df.columns:
        col_profile = {
            "name": str(col),
            "dtype": str(df[col].dtype),
            "null_ratio": float(df[col].isnull().sum() / len(df)),
            "unique_ratio": float(df[col].nunique() / len(df)),
            "semantic_type": _get_semantic_type(col, df[col]),
        }

        # Add statistical info for numeric columns
        if pd.api.types.is_numeric_dtype(df[col]):
            # Convert numpy types to Python native types
            col_profile.update(
                {
                    "mean": float(df[col].mean())
                    if not pd.isna(df[col].mean())
                    else None,
                    "std": float(df[col].std()) if not pd.isna(df[col].std()) else None,
                    "min": float(df[col].min()) if not pd.isna(df[col].min()) else None,
                    "max": float(df[col].max()) if not pd.isna(df[col].max()) else None,
                }
            )

        profile["columns"].append(col_profile)

    return profile


def recommend_algorithm(source_semantic, ref_semantic, source_series, ref_series):
    """Recommend best algorithm based on semantic types and data characteristics."""
    # Email domains
    if source_semantic == "email" or ref_semantic == "email":
        return "JaroWinkler"

    # Phone numbers
    if source_semantic == "phone" or ref_semantic == "phone":
        return "WRatio"

    # Company names
    if source_semantic == "name_company" or ref_semantic == "name_company":
        return "TokenSetRatio"

    # Person names
    if source_semantic == "name_person" or ref_semantic == "name_person":
        return "JaroWinkler"

    # IDs and codes
    if source_semantic == "id" or ref_semantic == "id":
        return "Exact"

    # Default
    return "WRatio"


def test_algorithms_on_sample(source_series, ref_series, profile):
    """Test different algorithms and return scores."""
    from fmatch.core.engine import ALGORITHM_MAP

    # Sample some pairs
    sample_size = min(100, len(source_series), len(ref_series))
    source_sample = source_series.sample(n=sample_size, random_state=42)
    ref_sample = ref_series.sample(n=sample_size, random_state=42)

    scores = {}
    for algo_name, algo_func in ALGORITHM_MAP.items():
        if algo_name == "Soundex":  # Skip for non-Latin
            if profile.get("script") not in ["latin", "unknown"]:
                continue

        # Calculate separation scores
        algo_scores = []
        for s, r in zip(source_sample, ref_sample):
            try:
                score = algo_func(str(s), str(r))
                algo_scores.append(score)
            except:
                pass

        if algo_scores:
            # Good algorithms have high variance (good discrimination)
            variance = np.var(algo_scores)
            scores[algo_name] = variance / 2500  # Normalize

    return scores


def _get_algo_for_field(field_name: str) -> str:
    """Get recommended algorithm based on field name."""
    field_lower = field_name.lower()

    if "email" in field_lower:
        return "exact"
    elif "domain" in field_lower or "website" in field_lower:
        return "domain"
    elif "company" in field_lower or "account" in field_lower or "name" in field_lower:
        return "token_set_ratio"
    elif "phone" in field_lower or "tel" in field_lower:
        return "exact"
    elif "id" in field_lower or "key" in field_lower:
        return "exact"
    else:
        return "token_set_ratio"  # Default


def _get_semantic_type(column_name: str, series: pd.Series) -> str:
    """Detect semantic type of a column"""
    col_lower = column_name.lower()

    # Check column name patterns
    if any(x in col_lower for x in ["id", "key", "code", "identifier"]):
        return "id"
    elif any(x in col_lower for x in ["email", "e-mail", "mail"]):
        return "email"
    elif any(x in col_lower for x in ["phone", "tel", "mobile", "cell"]):
        return "phone"
    elif any(x in col_lower for x in ["address", "street", "city", "state", "zip"]):
        return "address"
    elif any(x in col_lower for x in ["name", "first", "last", "surname"]):
        return "name"
    elif any(x in col_lower for x in ["company", "org", "business", "account"]):
        return "company"
    elif any(x in col_lower for x in ["date", "time", "created", "modified"]):
        return "date"
    elif any(x in col_lower for x in ["url", "link", "website"]):
        return "url"
    elif any(x in col_lower for x in ["domain", "host"]):
        return "domain"

    # Check data patterns
    if series.dtype in ["int64", "float64"]:
        return "numeric"

    # Sample data for pattern matching
    sample = series.dropna().head(10).astype(str)
    if len(sample) > 0:
        # Email pattern
        if sample.str.contains(r"@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}").any():
            return "email"
        # URL pattern
        if sample.str.contains(r"^https?://").any():
            return "url"
        # Phone pattern
        if sample.str.contains(r"^\+?\d{10,15}$").any():
            return "phone"

    return "text"


def _get_column_icon(semantic_type: str) -> str:
    """Get UI icon for column type"""
    icons = {
        "id": "🔑",
        "email": "📧",
        "phone": "📞",
        "address": "📍",
        "name": "👤",
        "company": "🏢",
        "date": "📅",
        "numeric": "🔢",
        "text": "📝",
        "url": "🔗",
        "domain": "🌐",
    }
    return icons.get(semantic_type, "ðŸ“Š")


def _get_column_color(semantic_type: str) -> str:
    """Get UI color for column type"""
    colors = {
        "id": "#FFD700",  # Gold
        "email": "#4169E1",  # Royal Blue
        "phone": "#32CD32",  # Lime Green
        "address": "#FF6347",  # Tomato
        "name": "#9370DB",  # Medium Purple
        "company": "#20B2AA",  # Light Sea Green
        "date": "#FF69B4",  # Hot Pink
        "numeric": "#FFA500",  # Orange
        "text": "#87CEEB",  # Sky Blue
        "url": "#00CED1",  # Dark Turquoise
        "domain": "#FF1493",  # Deep Pink
    }
    return colors.get(semantic_type, "#808080")  # Gray default


def _get_column_tooltip(column_name: str, series: pd.Series) -> str:
    """Generate helpful tooltip for column"""
    semantic_type = _get_semantic_type(column_name, series)
    fill_rate = series.notna().mean()
    unique_ratio = series.nunique() / len(series) if len(series) > 0 else 0

    if semantic_type == "id":
        return f"Likely ID column ({fill_rate:.0%} filled, {unique_ratio:.0%} unique)"
    elif semantic_type == "email":
        return f"Email addresses ({fill_rate:.0%} filled)"
    elif semantic_type == "phone":
        return f"Phone numbers ({fill_rate:.0%} filled)"
    elif unique_ratio > 0.9:
        return f"High cardinality column ({unique_ratio:.0%} unique)"
    elif unique_ratio < 0.1:
        return f"Low cardinality column ({unique_ratio:.0%} unique)"
    else:
        return f"{semantic_type.title()} column ({fill_rate:.0%} filled)"


def select_balanced_algorithm(profile):
    """Select algorithm balancing speed and accuracy."""
    semantic_type = profile.get("semantic_type", "text")

    if semantic_type == "email":
        return "JaroWinkler"
    elif semantic_type == "company" or semantic_type == "name_company":
        return "TokenSetRatio"
    elif semantic_type == "phone":
        return "WRatio"
    elif semantic_type == "id":
        return "Exact"
    else:
        return "WRatio"  # Good default balance


def select_accurate_algorithm(profile):
    """Select most accurate algorithm regardless of speed."""
    semantic_type = profile.get("semantic_type", "text")

    if semantic_type == "email":
        return "JaroWinkler"
    elif semantic_type == "company" or semantic_type == "name_company":
        return "TokenSetRatio"
    else:
        return "JaroWinkler"  # Most accurate for general text


def estimate_processing_time(source_rows, ref_rows, blocking, config):
    """Estimate processing time in seconds."""
    total_comparisons = (
        source_rows * ref_rows if ref_rows else source_rows * source_rows
    )

    if blocking and blocking.get("apply_blocking"):
        reduction = blocking.get("reduction_ratio", 0.9)
        total_comparisons *= 1 - reduction

    # Rough estimate: 1M comparisons per second
    return max(1, int(total_comparisons / 1_000_000))


def calculate_config_confidence(config, source_df, ref_df):
    """Calculate confidence score for the configuration."""
    confidence = 0.8  # Base confidence

    # Adjust based on mappings
    if config.get("mappings"):
        confidence += 0.1 if len(config["mappings"]) > 3 else 0.05

    # Adjust based on blocking
    if config.get("blocking_strategy"):
        confidence += 0.1

    return min(1.0, confidence)


def validate_configuration(config, source_df, ref_df):
    """Validate configuration and return warnings."""
    warnings = []

    # Check if datasets are very large without blocking
    if source_df is not None:
        total_comparisons = len(source_df) * (
            len(ref_df) if ref_df is not None else len(source_df)
        )
        if total_comparisons > 1_000_000 and not config.get("blocking_strategy"):
            warnings.append("Large dataset without blocking may be slow")

    # Check if mappings are present
    if not config.get("mappings"):
        warnings.append("No field mappings configured")

    return warnings


def create_column_profile(source_series, ref_series):
    """Create profile for algorithm selection."""
    profile = {
        "script": "latin",  # Detect script
        "semantic_type": _get_semantic_type(source_series.name, source_series),
        "avg_length": source_series.astype(str).str.len().mean(),
        "has_emoji": False,  # Detect emoji
        "encoding_issues": False,  # Detect encoding issues
    }
    return profile


def estimate_performance(source_rows, ref_rows, blocking_strategy, mode):
    """Estimate processing time and resources."""
    if mode == BlockingMode.MATCH:
        total_comparisons = source_rows * ref_rows
    else:
        total_comparisons = source_rows * (source_rows - 1) / 2

    # Apply blocking reduction
    if blocking_strategy.get("apply_blocking"):
        reduction = blocking_strategy.get("raw_metrics", {}).get("RR", 0.9)
        total_comparisons *= 1 - reduction

    # Estimate time (rough: 1M comparisons per second)
    estimated_seconds = max(1, total_comparisons / 1_000_000)

    # Estimate memory
    estimated_memory_mb = (source_rows + ref_rows) * 0.001  # 1KB per row

    return {
        "estimated_comparisons": int(total_comparisons),
        "estimated_time_seconds": int(estimated_seconds),
        "estimated_memory_mb": int(estimated_memory_mb),
        "recommended_workers": min(4, max(1, int(estimated_memory_mb / 500))),
    }


@router.get("/policy")
async def get_policy(
    segment: str | None = None,
    account_id: str = Depends(require_account),
    db: AsyncSession = Depends(get_session),
):
    store = PolicyStore(db)
    pol = await store.get_policy(str(account_id), segment or "default")
    return pol


_HASH_IGNORE_TOP_KEYS = {"hash", "config_hash"}


def _stable_intelligence_hash(payload: Dict[str, Any]) -> str:
    """Return a stable hash for the intelligence configuration."""

    def _json_default(obj: Any) -> Any:
        if isinstance(obj, pd.Timestamp):
            return obj.isoformat()
        if isinstance(obj, np.generic):
            try:
                return obj.item()
            except Exception:
                return str(obj)
        if hasattr(obj, "isoformat"):
            try:
                return obj.isoformat()
            except Exception:
                pass
        if isinstance(obj, (set, tuple)):
            return list(obj)
        return str(obj)

    data = deepcopy(payload)

    for key in list(data.keys()):
        if key in _HASH_IGNORE_TOP_KEYS:
            data.pop(key, None)

    options = data.get("options") if isinstance(data.get("options"), dict) else None
    raw_maps = []
    if isinstance(data.get("mappings"), list):
        raw_maps = data.get("mappings")
    elif options:
        raw_maps = options.get("maps") or options.get("column_maps") or []

    normalized_maps: list[Dict[str, Any]] = []
    for mapping in raw_maps:
        if not isinstance(mapping, dict):
            continue
        src = mapping.get("source")
        ref = mapping.get("reference") or mapping.get("ref")
        if not src or not ref:
            continue
        normalized_maps.append({"source": src, "reference": ref})
    normalized_maps.sort(key=lambda m: (m["source"], m["reference"]))

    data["mappings"] = normalized_maps
    if options is not None:
        options = dict(options)
        options["maps"] = normalized_maps
        options.pop("column_maps", None)
        data["options"] = options

    if "threshold" in data and isinstance(data["threshold"], float):
        data["threshold"] = round(float(data["threshold"]), 6)

    return hashlib.sha256(
        json.dumps(data, sort_keys=True, default=_json_default).encode("utf-8")
    ).hexdigest()
