"""
FoundryMatch SaaS - Smart Auto-Detector
========================================
This module provides an ML-driven service for automatically detecting
optimal matching configurations, including ID columns, field mappings,
and blocking strategies.
"""

import logging
import os
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

if os.getenv("FM_DISABLE_MLFLOW", "0") == "1":
    mlflow = None  # type: ignore[assignment]
else:  # pragma: no cover - exercised indirectly
    try:
        import mlflow  # type: ignore[import]
    except Exception:  # pragma: no cover - fallback path when mlflow missing/broken
        mlflow = None  # type: ignore[assignment]
import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator
from rapidfuzz import fuzz
from fmatch.core.engine import _get_individual_column_stats

log = logging.getLogger(__name__)


@dataclass
class SmartDetectionResult:
    """
    A structured result containing the auto-detected configuration for a matching job.
    """

    mode: str
    source_id_col: Optional[str] = None
    ref_id_col: Optional[str] = None
    mappings: List[Dict[str, Any]] = field(default_factory=list)
    blocking_strategy: Dict[str, Any] = field(default_factory=dict)
    confidence: float = 0.0
    model_versions_used: Dict[str, str] = field(default_factory=dict)


class SmartAutoDetector:
    """
    Uses pre-trained ML models and heuristics to automatically determine the best
    configuration for a fuzzy matching or deduplication job.
    """

    def __init__(self, mlflow_tracking_uri: Optional[str] = None):
        """
        Initializes the detector.

        Args:
            mlflow_tracking_uri: The URI for the MLflow tracking server.
                                 If None, models will not be loaded.
        """
        self.mlflow_tracking_uri = mlflow_tracking_uri
        self.models: Dict[str, BaseEstimator] = {}
        self._mlflow_available = False

        if mlflow is None:
            log.info(
                "MLflow disabled or unavailable; SmartAutoDetector running in heuristic-only mode."
            )
            return

        # Only set tracking URI if we can connect (avoid repeated connection attempts)
        if self.mlflow_tracking_uri:
            try:
                import socket
                import urllib.parse

                # Parse the URI to get host and port
                parsed = urllib.parse.urlparse(self.mlflow_tracking_uri)
                host = parsed.hostname or "127.0.0.1"
                port = parsed.port or 5000

                # Quick connectivity check with 1 second timeout
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.settimeout(1)
                result = sock.connect_ex((host, port))
                sock.close()

                if result == 0:
                    mlflow.set_tracking_uri(self.mlflow_tracking_uri)
                    self._mlflow_available = True
                    log.info(
                        f"MLflow tracking server accessible at {self.mlflow_tracking_uri}"
                    )
                else:
                    log.info(
                        f"MLflow server at {self.mlflow_tracking_uri} not accessible (port {port} closed) - ML features disabled"
                    )
            except Exception as e:
                log.info(f"MLflow not available ({e}) - continuing without ML features")
                self._mlflow_available = False

    def load_models(self, model_names: Optional[Dict[str, str]] = None):
        """
        Loads models from the MLflow Model Registry.

        Args:
            model_names: A dictionary mapping model roles (e.g., "id_detector")
                         to their registered model names in MLflow (e.g., "prod-id-detector").
        """
        if not self.mlflow_tracking_uri or not self._mlflow_available:
            log.debug("MLflow not available. Skipping model loading.")
            return
        if mlflow is None:
            log.debug("MLflow disabled. Skipping model loading.")
            return

        default_models = {
            "id_detector": "fm-id-column-classifier/production",
            "mapping_scorer": "fm-mapping-scorer/production",
            "algorithm_selector": "fm-algorithm-selector/production",
        }
        if model_names:
            default_models.update(model_names)

        log.info("Loading smart detection models from MLflow...")
        for role, model_uri in default_models.items():
            try:
                self.models[role] = mlflow.pyfunc.load_model(f"models:/{model_uri}")  # type: ignore[union-attr]
                log.info(f"  - Successfully loaded '{role}' model from '{model_uri}'.")
            except mlflow.exceptions.MlflowException as e:  # type: ignore[union-attr]
                log.error(f"  - Failed to load '{role}' model from '{model_uri}': {e}")

    def _featurize_column(self, series: pd.Series) -> Dict[str, Any]:
        """
        Generates a feature vector for a single column based on its statistics.
        This is used as input for the ML models.
        """
        stats = _get_individual_column_stats(series)
        features = {
            "fill_rate": stats.get("fill_rate", 0.0),
            "distinct_ratio": stats.get("distinct_ratio", 0.0),
            "median_char_length": stats.get("median_char_length", 0.0),
            "median_token_count": stats.get("median_token_count", 0.0),
            "is_numeric": 1 if stats.get("inferred_type") == "numeric" else 0,
            "is_datetime": 1 if stats.get("inferred_type") == "datetime" else 0,
        }
        return features

    def detect_configuration(
        self, source_df: pd.DataFrame, ref_df: Optional[pd.DataFrame] = None
    ) -> Dict[str, Any]:
        """
        Orchestrates the entire detection process for a given dataset.

        Args:
            source_df: The source DataFrame.
            ref_df: The reference DataFrame (for match mode).

        Returns:
            A SmartDetectionResult object with the suggested configuration.
        """
        # Determine mode based on whether reference DataFrame is provided
        mode = "match" if ref_df is not None else "dedupe"

        if not self.models:
            log.warning("No models loaded. Cannot perform smart detection.")
            return SmartDetectionResult(mode="error")

        log.info(f"Starting smart configuration detection for '{mode}' mode.")

        source_id_col = self._detect_id_column(source_df, "source")
        ref_id_col = (
            self._detect_id_column(ref_df, "reference") if mode == "match" else None
        )

        suggested_mappings = (
            self._generate_intelligent_mappings(source_df, ref_df)
            if ref_df is not None
            else []
        )
        mappings_with_algos = suggested_mappings  # Already includes algorithm selection
        blocking_strategy = self._determine_blocking_strategy(
            source_df, ref_df, mappings_with_algos
        )

        # Calculate overall confidence
        confidence = 0.85  # Placeholder confidence - could be enhanced with actual confidence calculation

        # Prepare model versions used
        model_versions = {}
        if self.models:
            model_versions = {
                role: model.metadata.model_uuid for role, model in self.models.items()
            }
        else:
            model_versions = {"detector": "v1.0", "heuristics": "v2.0"}

        return SmartDetectionResult(
            mode=mode,
            source_id_col=source_id_col,
            ref_id_col=ref_id_col,
            mappings=mappings_with_algos,
            blocking_strategy=blocking_strategy,
            confidence=confidence,
            model_versions_used=model_versions,
        )

    def _detect_id_column(
        self, df: Optional[pd.DataFrame], role: str = "source"
    ) -> Optional[str]:
        """Detect ID column using heuristics and ML models if available"""
        if df is None or df.empty:
            return None

        # First try ML model if available
        if "id_detector" in self.models:
            ml_result = self._predict_id_column_ml(df, role)
            if ml_result:
                return ml_result

        # Fall back to heuristics
        candidates = []

        for col in df.columns:
            col_lower = col.lower()
            score = 0

            # Check common ID patterns
            id_patterns = [
                "_id",
                "id_",
                "identifier",
                "key",
                "code",
                "number",
                "no",
                "num",
            ]
            for pattern in id_patterns:
                if pattern in col_lower:
                    score += 10

            # Check if column name is exactly 'id'
            if col_lower == "id":
                score += 20

            # Check uniqueness
            if len(df[col].dropna()) > 0:
                uniqueness = df[col].nunique() / len(df[col].dropna())
                if uniqueness > 0.95:
                    score += 15
                elif uniqueness > 0.90:
                    score += 10
                elif uniqueness > 0.80:
                    score += 5

            # Check data type (prefer strings/objects for IDs)
            if df[col].dtype == "object":
                score += 5
            elif df[col].dtype in ["int64", "int32"]:
                score += 3

            # Check if values look like IDs (alphanumeric patterns)
            sample_values = df[col].dropna().head(100).astype(str)
            if len(sample_values) > 0:
                # Check for consistent patterns (e.g., "ID-12345", "CUST_001")
                if any("-" in str(v) or "_" in str(v) for v in sample_values):
                    score += 5

                # Check for numeric sequences
                try:
                    numeric_vals = pd.to_numeric(sample_values, errors="coerce")
                    if numeric_vals.notna().sum() > len(sample_values) * 0.8:
                        # Mostly numeric - check if sequential
                        if len(numeric_vals.dropna()) > 1:
                            diffs = numeric_vals.dropna().sort_values().diff()
                            if (diffs == 1).sum() > len(diffs) * 0.5:
                                score += 10  # Likely sequential IDs
                except:
                    pass

            candidates.append((col, score))

        # Sort by score and return best candidate
        candidates.sort(key=lambda x: x[1], reverse=True)

        if candidates and candidates[0][1] > 10:
            best_col = candidates[0][0]
            log.info(
                f"Detected '{best_col}' as {role} ID column (score: {candidates[0][1]})"
            )
            return best_col

        log.warning(f"No suitable ID column found for {role} DataFrame")
        return None

    def _predict_id_column_ml(self, df: pd.DataFrame, role: str) -> Optional[str]:
        """Uses ML model to predict ID column if available"""
        try:
            log.info(f"Using ML model to predict ID column for '{role}' DataFrame...")
            features_list = [self._featurize_column(df[col]) for col in df.columns]
            if not features_list:
                return None

            features_df = pd.DataFrame(features_list)
            predictions = self.models["id_detector"].predict(features_df)
            best_candidate_idx = pd.Series(predictions).idxmax()
            best_column_name = df.columns[best_candidate_idx]
            log.info(f"  - ML model selected '{best_column_name}' as {role} ID")
            return best_column_name
        except Exception as e:
            log.debug(f"ML ID column prediction failed: {e}")
            return None

    def _generate_intelligent_mappings(
        self, source_df: pd.DataFrame, ref_df: pd.DataFrame
    ) -> List[Dict]:
        """Generate mappings using semantic similarity and data patterns"""
        mappings = []
        used_ref_cols = set()

        for src_col in source_df.columns:
            best_match = None
            best_score = 0
            best_algo = "WRatio"

            for ref_col in ref_df.columns:
                if ref_col in used_ref_cols:
                    continue

                # 1. Name similarity (40% weight)
                name_score = self._calculate_name_similarity(src_col, ref_col)

                # 2. Data type compatibility (20% weight)
                type_score = self._check_type_compatibility(
                    source_df[src_col], ref_df[ref_col]
                )

                # 3. Statistical similarity for numeric columns (20% weight)
                stat_score = 0
                if source_df[src_col].dtype in ["int64", "float64"] and ref_df[
                    ref_col
                ].dtype in ["int64", "float64"]:
                    stat_score = self._calculate_statistical_similarity(
                        source_df[src_col], ref_df[ref_col]
                    )

                # 4. Value overlap similarity (20% weight)
                overlap_score = self._calculate_value_overlap(
                    source_df[src_col], ref_df[ref_col]
                )

                # Combined weighted score
                combined_score = (
                    name_score * 0.4
                    + type_score * 0.2
                    + stat_score * 0.2
                    + overlap_score * 0.2
                )

                if (
                    combined_score > best_score and combined_score >= 50
                ):  # Minimum threshold
                    best_score = combined_score
                    best_match = ref_col
                    best_algo = self._select_algorithm_for_pair(
                        source_df[src_col], ref_df[ref_col]
                    )

            if best_match:
                mappings.append(
                    {
                        "source": src_col,
                        "ref": best_match,
                        "weight": best_score / 100.0,
                        "preferred_algo": best_algo,
                        "algo_confidence": best_score / 100.0,
                        "auto_mapped": True,
                        "similarity_score": best_score,
                    }
                )
                used_ref_cols.add(best_match)
                log.debug(
                    f"Mapped '{src_col}' -> '{best_match}' (score: {best_score:.1f}, algo: {best_algo})"
                )

        return mappings

    def _calculate_name_similarity(self, name1: str, name2: str) -> float:
        """Calculate semantic similarity between column names"""
        # Normalize names
        norm1 = name1.lower().replace("_", " ").replace("-", " ")
        norm2 = name2.lower().replace("_", " ").replace("-", " ")

        # Try multiple fuzzy matching algorithms
        scores = [
            fuzz.ratio(norm1, norm2),
            fuzz.token_set_ratio(norm1, norm2),
            fuzz.token_sort_ratio(norm1, norm2),
            fuzz.partial_ratio(norm1, norm2),
        ]

        # Use weighted average, favoring token_set_ratio for flexibility
        weighted_score = (
            scores[0] * 0.2  # Exact ratio
            + scores[1] * 0.4  # Token set (handles word order variations)
            + scores[2] * 0.2  # Token sort
            + scores[3] * 0.2  # Partial match
        )

        return weighted_score

    def _check_type_compatibility(
        self, series1: pd.Series, series2: pd.Series
    ) -> float:
        """Check if two columns have compatible data types"""
        dtype1 = str(series1.dtype)
        dtype2 = str(series2.dtype)

        # Exact match
        if dtype1 == dtype2:
            return 100.0

        # Compatible numeric types
        numeric_types = ["int64", "int32", "float64", "float32"]
        if dtype1 in numeric_types and dtype2 in numeric_types:
            return 90.0

        # String and object are compatible
        if dtype1 in ["object", "string"] and dtype2 in ["object", "string"]:
            return 95.0

        # Numeric can be compared with string (after conversion)
        if (dtype1 in numeric_types and dtype2 in ["object", "string"]) or (
            dtype2 in numeric_types and dtype1 in ["object", "string"]
        ):
            return 70.0

        # Date/datetime types
        datetime_types = ["datetime64", "datetime64[ns]", "date"]
        if any(dt in dtype1 for dt in datetime_types) and any(
            dt in dtype2 for dt in datetime_types
        ):
            return 95.0

        # Otherwise incompatible
        return 0.0

    def _calculate_statistical_similarity(
        self, series1: pd.Series, series2: pd.Series
    ) -> float:
        """Calculate statistical similarity for numeric columns"""
        try:
            # Remove nulls
            s1 = series1.dropna()
            s2 = series2.dropna()

            if len(s1) < 2 or len(s2) < 2:
                return 0.0

            # Normalize to [0, 1] range for comparison
            s1_norm = (s1 - s1.min()) / (s1.max() - s1.min() + 1e-10)
            s2_norm = (s2 - s2.min()) / (s2.max() - s2.min() + 1e-10)

            # Compare distributions
            scores = []

            # 1. Mean similarity
            mean_diff = abs(s1_norm.mean() - s2_norm.mean())
            scores.append(max(0, 100 * (1 - mean_diff)))

            # 2. Standard deviation similarity
            std_diff = abs(s1_norm.std() - s2_norm.std())
            scores.append(max(0, 100 * (1 - std_diff)))

            # 3. Range similarity
            range1 = s1.max() - s1.min()
            range2 = s2.max() - s2.min()
            if range1 > 0 and range2 > 0:
                range_ratio = min(range1, range2) / max(range1, range2)
                scores.append(100 * range_ratio)

            # 4. Quartile similarity
            try:
                q1_1, q2_1, q3_1 = s1_norm.quantile([0.25, 0.5, 0.75])
                q1_2, q2_2, q3_2 = s2_norm.quantile([0.25, 0.5, 0.75])

                quartile_diff = (
                    abs(q1_1 - q1_2) + abs(q2_1 - q2_2) + abs(q3_1 - q3_2)
                ) / 3
                scores.append(max(0, 100 * (1 - quartile_diff)))
            except:
                pass

            return np.mean(scores) if scores else 0.0

        except Exception as e:
            log.debug(f"Statistical similarity calculation failed: {e}")
            return 0.0

    def _calculate_value_overlap(self, series1: pd.Series, series2: pd.Series) -> float:
        """Calculate how much actual values overlap between columns"""
        try:
            # Convert to string for comparison
            s1_values = set(series1.dropna().astype(str).str.lower().str.strip())
            s2_values = set(series2.dropna().astype(str).str.lower().str.strip())

            if not s1_values or not s2_values:
                return 0.0

            # Calculate Jaccard similarity
            intersection = len(s1_values & s2_values)
            union = len(s1_values | s2_values)

            if union == 0:
                return 0.0

            jaccard = intersection / union

            # Also consider partial matches for strings
            if series1.dtype == "object" and series2.dtype == "object":
                partial_matches = 0
                sample_s1 = list(s1_values)[:100]  # Sample for performance
                sample_s2 = list(s2_values)[:100]

                for v1 in sample_s1:
                    for v2 in sample_s2:
                        if len(v1) > 3 and len(v2) > 3:  # Only for meaningful strings
                            if fuzz.partial_ratio(v1, v2) > 85:
                                partial_matches += 1
                                break

                partial_score = partial_matches / len(sample_s1) if sample_s1 else 0
                return 100 * (jaccard * 0.7 + partial_score * 0.3)

            return 100 * jaccard

        except Exception as e:
            log.debug(f"Value overlap calculation failed: {e}")
            return 0.0

    def _select_algorithm_for_column(self, series: pd.Series) -> str:
        """Select best algorithm based on column characteristics"""
        if series.dtype in ["int64", "int32", "float64", "float32"]:
            return "Exact"  # Numeric columns should use exact matching

        # For string columns, analyze characteristics
        str_series = series.dropna().astype(str)
        if len(str_series) == 0:
            return "WRatio"

        # Calculate average length
        avg_length = str_series.str.len().mean()

        # Check for specific patterns
        sample = str_series.head(100)

        # Email addresses
        if sample.str.contains("@").sum() > len(sample) * 0.5:
            return "Exact"  # Emails should match exactly

        # Phone numbers (various formats)
        phone_pattern = r"[\d\-\(\)\+\s]{7,}"
        if sample.str.match(phone_pattern).sum() > len(sample) * 0.5:
            return "JaroWinkler"  # Good for phone numbers with minor variations

        # Short codes or IDs
        if avg_length < 10 and str_series.str.isalnum().sum() > len(str_series) * 0.8:
            return "JaroWinkler"  # Good for short alphanumeric codes

        # Long text fields
        if avg_length > 50:
            return "TokenSetRatio"  # Good for long text with varying word order

        # Names (person or company)
        if avg_length >= 10 and avg_length <= 30:
            # Check if mostly alphabetic with spaces
            if sample.str.replace(" ", "").str.isalpha().sum() > len(sample) * 0.7:
                return "WRatio"  # Good for names with minor variations

        # Default fallback
        return "WRatio"  # Good general-purpose algorithm

    def _select_algorithm_for_pair(self, series1: pd.Series, series2: pd.Series) -> str:
        """Select best algorithm for a specific column pair"""
        # Get individual recommendations
        algo1 = self._select_algorithm_for_column(series1)
        algo2 = self._select_algorithm_for_column(series2)

        # If both agree, use that
        if algo1 == algo2:
            return algo1

        # If one is Exact and types are compatible, prefer Exact
        if "Exact" in [algo1, algo2]:
            if self._check_type_compatibility(series1, series2) > 80:
                return "Exact"

        # For mixed recommendations, use WRatio as safe default
        return "WRatio"

    def _determine_blocking_strategy(
        self,
        source_df: pd.DataFrame,
        ref_df: Optional[pd.DataFrame],
        mappings: List[Dict],
    ) -> Dict[str, Any]:
        """Determine optimal blocking strategy using actual metrics"""
        try:
            # Use the core engine's blocking strategy determination
            from fmatch.core.engine import determine_optimal_blocking_strategy

            # Extract column names from mappings
            if mappings and isinstance(mappings, list):
                source_cols = [m["source"] for m in mappings if "source" in m]
                ref_cols = (
                    [m.get("ref", m.get("reference", m["source"])) for m in mappings]
                    if ref_df is not None
                    else source_cols
                )

                # Filter to only columns that exist in the DataFrames
                source_cols = [col for col in source_cols if col in source_df.columns]
                if ref_df is not None:
                    ref_cols = [col for col in ref_cols if col in ref_df.columns]

                if source_cols:
                    # Pass the actual DataFrame subsets, not the full DataFrames
                    # The function expects DataFrames, not column lists
                    source_subset = source_df[source_cols] if source_cols else source_df
                    ref_subset = (
                        ref_df[ref_cols]
                        if ref_df is not None and ref_cols
                        else source_subset
                    )

                    # Call the engine's blocking strategy function with DataFrames
                    strategy_result = determine_optimal_blocking_strategy(
                        source_subset,
                        ref_subset,
                        mappings,  # Pass mappings as metadata
                    )

                    # Convert to our format
                    return {
                        "apply_blocking": True,
                        "source_block_col": strategy_result.get("source_block_col"),
                        "ref_block_col": strategy_result.get("ref_block_col"),
                        "preproc": strategy_result.get("preproc", "first4"),
                        "block_key_len": strategy_result.get("block_key_len", 4),
                        "quality_score": strategy_result.get("quality_score", 0.75),
                        "entropy": strategy_result.get("entropy", 0),
                        "reduction_ratio": strategy_result.get("reduction_ratio", 0.95),
                        "metrics": strategy_result,
                    }

            # Fallback if no mappings or columns
            return self._fallback_blocking_strategy(source_df, ref_df)

        except (ImportError, AttributeError, KeyError, TypeError) as e:
            log.warning(f"Failed to determine blocking strategy: {e}")
            return self._fallback_blocking_strategy(source_df, ref_df)

    def _fallback_blocking_strategy(
        self, source_df: pd.DataFrame, ref_df: Optional[pd.DataFrame]
    ) -> Dict[str, Any]:
        """Provide a sensible fallback blocking strategy when automatic detection fails"""
        # Fallback strategy - use first column as blocking column
        source_block_col = None
        ref_block_col = None

        if source_df is not None and not source_df.empty and len(source_df.columns) > 0:
            # Try to find a good blocking column - prefer string columns with reasonable cardinality
            for col in source_df.columns:
                if source_df[col].dtype == "object":  # String column
                    cardinality = source_df[col].nunique()
                    if (
                        10 < cardinality < len(source_df) * 0.5
                    ):  # Not too few, not too many unique values
                        source_block_col = col
                        break

            # If no good string column found, use first column
            if source_block_col is None:
                source_block_col = source_df.columns[0]

        if ref_df is not None and not ref_df.empty and len(ref_df.columns) > 0:
            # Try to use same column name if it exists
            if source_block_col and source_block_col in ref_df.columns:
                ref_block_col = source_block_col
            else:
                # Otherwise use first column
                ref_block_col = ref_df.columns[0]
        else:
            ref_block_col = source_block_col

        return {
            "apply_blocking": bool(source_block_col),
            "source_block_col": source_block_col,
            "ref_block_col": ref_block_col,
            "preproc": "first4",
            "block_key_len": 4,
            "quality_score": 0.5,
            "fallback": True,
        }

    def _calculate_confidence(
        self, mappings: List[Dict], blocking_strategy: Dict
    ) -> float:
        """Calculate overall confidence score for the configuration"""
        scores = []

        # Mapping confidence (40% weight)
        if mappings:
            mapping_scores = [m.get("weight", 0) for m in mappings]
            avg_mapping_confidence = np.mean(mapping_scores) if mapping_scores else 0
            scores.append(avg_mapping_confidence * 0.4)
        else:
            scores.append(0)

        # Blocking strategy confidence (30% weight)
        blocking_confidence = blocking_strategy.get("quality_score", 0.5)
        scores.append(blocking_confidence * 0.3)

        # Coverage confidence (30% weight)
        # Higher confidence if we have good coverage of columns
        if mappings:
            coverage = min(1.0, len(mappings) / 5)  # Assume 5 mappings is good coverage
            scores.append(coverage * 0.3)
        else:
            scores.append(0.15)  # Some confidence even without mappings

        return sum(scores)

    def _featurize_column(self, series: pd.Series) -> Dict[str, Any]:
        """Extract features from a column for ML models"""
        features = {
            "name_length": len(series.name),
            "has_id_in_name": "id" in series.name.lower(),
            "uniqueness_ratio": series.nunique() / len(series)
            if len(series) > 0
            else 0,
            "null_ratio": series.isnull().sum() / len(series) if len(series) > 0 else 0,
            "dtype_numeric": 1 if series.dtype in ["int64", "float64"] else 0,
            "dtype_string": 1 if series.dtype == "object" else 0,
            "avg_length": series.astype(str).str.len().mean()
            if series.dtype == "object"
            else 0,
            "is_sequential": 0,  # Would need more complex logic to determine
        }
        return features
