"""
Google Sheets integration API endpoints.
Provides direct data processing for Google Sheets Add-on.
"""

from typing import List, Dict, Any, Optional, Tuple
from uuid import UUID, uuid4
from dataclasses import asdict, is_dataclass
import pandas as pd
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from pydantic import BaseModel, Field
import logging
from collections import Counter, defaultdict

from ..auth import get_current_account
from ..models import Account, JobStatus
from ..repositories.job_repo import JobRepository
from ..ml.smart_detection import SmartAutoDetector
from ..services.manual_match_service import (
    MappingConfig,
    MappingOptions,
    execute as manual_match_execute,
    _extract_domain,
    _extract_email_domain,
)
from ..services.domain_family import get_registry
from ..middleware.quota_checker import (
    get_quota,
    record_quota_usage,
)

# Note: Background processing for Sheets is implemented inline in this module
# using process_sheets_dedupe_job/process_sheets_match_job. Avoid importing
# worker functions to prevent circular or missing import errors.
from ..db import get_session as get_db
from sqlalchemy.ext.asyncio import AsyncSession
import json

logger = logging.getLogger(__name__)

router = APIRouter(tags=["sheets"])


def _detection_to_dict(result: Any) -> Dict[str, Any]:
    """Normalize SmartAutoDetector output to a plain dictionary."""
    if result is None:
        return {}
    if isinstance(result, dict):
        return result
    if is_dataclass(result):
        return asdict(result)
    # Fallback to reading known attributes
    data: Dict[str, Any] = {}
    for attr in (
        "mode",
        "source_id_col",
        "ref_id_col",
        "mappings",
        "blocking_strategy",
        "confidence",
        "model_versions_used",
    ):
        if hasattr(result, attr):
            data[attr] = getattr(result, attr)
    return data


class SheetDedupeRequest(BaseModel):
    """Request model for sheet dedupe"""

    data: List[Dict[str, Any]] = Field(
        ..., description="Sheet data as list of dictionaries"
    )
    id_column: Optional[str] = Field(None, description="ID column name (optional)")
    use_intelligent_config: bool = Field(
        True, description="Use AI-powered configuration"
    )
    threshold: int = Field(85, ge=0, le=100, description="Matching threshold")
    sheet_name: Optional[str] = Field(
        None, description="Name of the sheet being processed"
    )
    row_map: Optional[Dict[str, int]] = Field(
        None, description="Mapping of record IDs to row numbers"
    )


class SheetMatchRequest(BaseModel):
    """Request model for sheet matching"""

    source_data: List[Dict[str, Any]] = Field(..., description="Source sheet data")
    reference_data: List[Dict[str, Any]] = Field(
        ..., description="Reference sheet data"
    )
    source_id_column: Optional[str] = Field(None, description="Source ID column")
    reference_id_column: Optional[str] = Field(None, description="Reference ID column")
    use_intelligent_config: bool = Field(
        True, description="Use AI-powered configuration"
    )
    threshold: int = Field(80, ge=0, le=100, description="Matching threshold")


class SheetProgressResponse(BaseModel):
    """Progress response for async sheet operations"""

    job_id: str
    status: str
    progress: float
    message: Optional[str] = None
    eta_seconds: Optional[float] = None
    rows_processed: Optional[int] = None
    total_rows: Optional[int] = None


@router.post("/dedupe")
async def sheets_dedupe(
    request: SheetDedupeRequest,
    background_tasks: BackgroundTasks,
    db: AsyncSession = Depends(get_db),
    account: Account = Depends(get_current_account),
) -> Dict[str, Any]:
    """
    Direct dedupe endpoint for Google Sheets.
    Accepts data directly as JSON and returns results synchronously for small datasets.
    """
    try:
        # Convert to DataFrame
        if not request.data:
            raise HTTPException(status_code=400, detail="No data provided")

        df = pd.DataFrame(request.data)
        logger.info(f"Processing sheet dedupe with {len(df)} rows")

        account_id_value = getattr(account, "id", account)
        quota = await get_quota(str(account_id_value), db)
        quota_check = quota.check_can_run(estimated_rows=len(df))
        if not quota_check.get("allowed", True):
            message = quota_check.get("message", "Quota limit reached")
            raise HTTPException(status_code=429, detail=message)

        # Determine if we should process async (large datasets)
        is_async = len(df) > 1000

        config: Dict[str, Any] = {}
        if request.use_intelligent_config:
            # Use SmartAutoDetector for intelligent configuration
            detector = SmartAutoDetector()
            try:
                detection_result = detector.detect_configuration(df)
            except Exception as exc:  # pragma: no cover - defensive guard
                logger.exception("Smart detection failed for Sheets dedupe: %s", exc)
                detection_result = {}

            detected_config = _detection_to_dict(detection_result)

            # Use detected ID column if not specified
            if not request.id_column:
                request.id_column = detected_config.get("source_id_col")

            # Get optimal mappings
            config = {
                "id_column": request.id_column,
                "mappings": detected_config.get("mappings", []),
                "blocking_strategy": detected_config.get("blocking_strategy", {}),
                "threshold": request.threshold,
            }

            logger.info(f"Using intelligent config: {config}")
        else:
            # Use simple configuration
            config = {"id_column": request.id_column, "threshold": request.threshold}

        if not config.get("id_column"):
            config["id_column"] = "__row_id__"

        if is_async:
            # Create a job for async processing
            job_repo = JobRepository(db)

            # Store data temporarily
            source_data = df.to_json(orient="records")

            job = await job_repo.create_job(
                job_id=str(uuid4()),
                project_id=None,
                account_id=str(account_id_value) if account_id_value else None,
                status=JobStatus.QUEUED.value,
                mode="dedupe",
                config={
                    "configuration": config,
                    "metadata": {
                        "source": "google_sheets",
                        "sheet_name": request.sheet_name or "Google Sheet",
                        "row_map": request.row_map,
                        "source_data": source_data,
                        "row_count": len(df),
                        "account_id": str(account_id_value)
                        if account_id_value
                        else None,
                    },
                },
            )

            # Process in background
            background_tasks.add_task(
                process_sheets_dedupe_job,
                job.id,
                df,
                config,
                str(account_id_value) if account_id_value else None,
                db,
            )

            return {
                "job_id": str(job.id),
                "status": "processing",
                "message": f"Processing {len(df)} rows asynchronously",
                "is_async": True,
            }
        else:
            # Process synchronously for small datasets
            results = await process_dedupe_inline(df, config)

            stats = results.get("stats") or {
                "total_records": len(df),
                "duplicate_groups": results.get("duplicate_groups", 0),
                "duplicates_found": results.get("duplicates_found", 0),
            }

            response = {
                "matches": results.get("matches", []),
                "duplicates": results.get("duplicates", []),
                "config_used": config,
                "stats": stats,
                "telemetry": results.get("telemetry", {}),
                "duplicate_groups": results.get(
                    "duplicate_groups", stats.get("duplicate_groups", 0)
                ),
                "duplicates_found": results.get(
                    "duplicates_found", stats.get("duplicates_found", 0)
                ),
                "is_async": False,
            }
            await record_quota_usage(
                str(account_id_value) if account_id_value else None,
                rows_returned=len(response["matches"]),
                db=db,
            )
            return response

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Sheet dedupe error: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))


@router.post("/match")
async def sheets_match(
    request: SheetMatchRequest,
    background_tasks: BackgroundTasks,
    db: AsyncSession = Depends(get_db),
    account: Account = Depends(get_current_account),
) -> Dict[str, Any]:
    """
    Direct match endpoint for Google Sheets.
    Matches two sheets of data.
    """
    try:
        # Convert to DataFrames
        source_df = pd.DataFrame(request.source_data)
        reference_df = pd.DataFrame(request.reference_data)

        logger.info(
            f"Processing sheet match: {len(source_df)} x {len(reference_df)} rows"
        )

        account_id_value = getattr(account, "id", account)
        quota = await get_quota(str(account_id_value), db)
        quota_check = quota.check_can_run(estimated_rows=len(source_df))
        if not quota_check.get("allowed", True):
            message = quota_check.get("message", "Quota limit reached")
            raise HTTPException(status_code=429, detail=message)

        # Determine if we should process async
        is_async = (len(source_df) * len(reference_df)) > 100000

        config = {}
        if request.use_intelligent_config:
            # Use SmartAutoDetector
            detector = SmartAutoDetector()
            try:
                detection_result = detector.detect_configuration(
                    source_df, ref_df=reference_df
                )
            except Exception as exc:  # pragma: no cover - defensive guard
                logger.exception("Smart detection failed for Sheets match: %s", exc)
                detection_result = {}

            detected_config = _detection_to_dict(detection_result)

            if not request.source_id_column:
                request.source_id_column = detected_config.get("source_id_col")
            if not request.reference_id_column:
                request.reference_id_column = detected_config.get("ref_id_col")

            config = {
                "source_id_column": request.source_id_column,
                "reference_id_column": request.reference_id_column,
                "mappings": detected_config.get("mappings", []),
                "blocking_strategy": detected_config.get("blocking_strategy", {}),
                "threshold": request.threshold,
            }
        else:
            config = {
                "source_id_column": request.source_id_column,
                "reference_id_column": request.reference_id_column,
                "mappings": [],
                "blocking_strategy": {},
                "threshold": request.threshold,
            }
        config.setdefault("mappings", [])
        config.setdefault("blocking_strategy", {})

        if is_async:
            # Create job for async processing
            job_repo = JobRepository(db)

            job = await job_repo.create_job(
                job_id=str(uuid4()),
                project_id=None,
                account_id=str(account_id_value) if account_id_value else None,
                status=JobStatus.QUEUED.value,
                mode="match",
                config={
                    "configuration": config,
                    "metadata": {
                        "source": "google_sheets",
                        "source_sheet_name": "Source Sheet",
                        "reference_sheet_name": "Reference Sheet",
                        "source_rows": len(source_df),
                        "reference_rows": len(reference_df),
                        "source_data": source_df.to_json(orient="records"),
                        "reference_data": reference_df.to_json(orient="records"),
                        "account_id": str(account_id_value)
                        if account_id_value
                        else None,
                    },
                },
            )

            # Process in background
            background_tasks.add_task(
                process_sheets_match_job,
                job.id,
                source_df,
                reference_df,
                config,
                str(account_id_value) if account_id_value else None,
                db,
            )

            return {
                "job_id": str(job.id),
                "status": "processing",
                "message": f"Processing {len(source_df)} x {len(reference_df)} matches asynchronously",
                "is_async": True,
            }
        else:
            # Process synchronously
            results = await process_match_inline(source_df, reference_df, config)

            response = {
                "matches": results["matches"],
                "config_used": config,
                "stats": {
                    "source_records": len(source_df),
                    "reference_records": len(reference_df),
                    "matches_found": len(results["matches"]),
                },
                "is_async": False,
            }
            await record_quota_usage(
                str(account_id_value) if account_id_value else None,
                rows_returned=len(results["matches"]),
                db=db,
            )
            return response

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Sheet match error: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/jobs/{job_id}/progress")
async def get_sheet_job_progress(
    job_id: UUID,
    db: AsyncSession = Depends(get_db),
    account: Account = Depends(get_current_account),
) -> SheetProgressResponse:
    """Get progress for an async sheet job"""
    job_repo = JobRepository(db)
    job = await job_repo.get_job(job_id, account.id)

    if not job:
        raise HTTPException(status_code=404, detail="Job not found")

    # Check if this is a sheets job
    metadata = job.metadata or {}
    if metadata.get("source") != "google_sheets":
        raise HTTPException(status_code=400, detail="Not a Google Sheets job")

    return SheetProgressResponse(
        job_id=str(job.id),
        status=job.status,
        progress=job.progress or 0,
        message=job.message,
        eta_seconds=job.estimated_duration_seconds,
        rows_processed=metadata.get("rows_processed"),
        total_rows=metadata.get("total_rows"),
    )


@router.get("/jobs/{job_id}/results")
async def get_sheet_job_results(
    job_id: UUID,
    db: AsyncSession = Depends(get_db),
    account: Account = Depends(get_current_account),
) -> Dict[str, Any]:
    """Get results for a completed sheet job"""
    job_repo = JobRepository(db)
    job = await job_repo.get_job(job_id, account.id)

    if not job:
        raise HTTPException(status_code=404, detail="Job not found")

    if job.status != JobStatus.COMPLETED:
        raise HTTPException(
            status_code=400, detail=f"Job is {job.status}, not completed"
        )

    # Return results with row mapping if available
    metadata = job.metadata or {}
    results = job.result_preview or []

    # If we have row_map, enhance results with original row numbers
    if metadata.get("row_map"):
        row_map = metadata["row_map"]
        for result in results:
            if "record_id" in result and result["record_id"] in row_map:
                result["original_row"] = row_map[result["record_id"]]

    return {
        "matches": results,
        "config_used": job.configuration,
        "stats": job.result_summary or {},
        "row_map": metadata.get("row_map"),
    }


class SheetsManualDedupe:
    def __init__(self, df: pd.DataFrame, config: Dict[str, Any]) -> None:
        self.config = config or {}
        self.df = df.copy().reset_index(drop=True)
        if "__row_id__" not in self.df.columns:
            self.df["__row_id__"] = self.df.index.astype(str)
        self.id_column = self.config.get("id_column")
        raw_threshold = self.config.get("threshold", 85)
        self.threshold = self._normalize_threshold(raw_threshold)
        self.threshold_pct = self.threshold * 100.0
        self.email_columns = self._detect_columns(["email"])
        self.domain_columns = self._detect_columns(["domain", "website", "url", "site"])
        self.name_columns = self._detect_name_columns()
        self.state_columns = self._detect_columns(["state", "province"])
        self.country_columns = self._detect_columns(["country"])
        self.display_column = self._select_display_column(
            ["company name", "account name", "name"]
        )
        self.mappings_used: List[MappingConfig] = []

    @staticmethod
    def _normalize_threshold(value: Any) -> float:
        try:
            threshold = float(value)
        except (TypeError, ValueError):
            threshold = 85.0
        if threshold > 1.0:
            threshold = threshold / 100.0
        return max(0.0, min(threshold, 1.0))

    @staticmethod
    def _normalize_weight(value: Any) -> float:
        try:
            weight = float(value)
        except (TypeError, ValueError):
            weight = 1.0
        if weight > 1.0:
            weight = weight / 100.0
        return max(0.05, min(weight, 1.0))

    def _detect_columns(self, keywords: List[str]) -> List[str]:
        results: List[str] = []
        lowered_keywords = [kw.lower() for kw in keywords]
        for column in self.df.columns:
            lowered = column.lower()
            if column == "__row_id__":
                continue
            if self.id_column and column == self.id_column:
                continue
            if any(keyword in lowered for keyword in lowered_keywords):
                results.append(column)
        return results

    def _detect_name_columns(self) -> List[str]:
        name_columns = self._detect_columns(
            ["company", "name", "account", "organization"]
        )
        seen = set()
        ordered: List[str] = []
        for col in name_columns:
            if col not in seen:
                ordered.append(col)
                seen.add(col)
        return ordered

    def _select_display_column(self, preferences: List[str]) -> Optional[str]:
        lowered = {col.lower(): col for col in self.df.columns}
        for pref in preferences:
            key = pref.lower()
            if key in lowered:
                return lowered[key]
        for col in self.df.columns:
            if "name" in col.lower():
                return col
        return None

    def _resolve_row_id(self, index: int) -> str:
        value = (
            self.df.at[index, "__row_id__"]
            if "__row_id__" in self.df.columns
            else index
        )
        if pd.isna(value):
            value = index
        return str(value)

    def _resolve_record_id(self, index: int) -> str:
        if self.id_column and self.id_column in self.df.columns:
            value = self.df.at[index, self.id_column]
            if pd.notna(value):
                return str(value)
        return self._resolve_row_id(index)

    def _resolve_display_value(self, index: int) -> str:
        if self.display_column and self.display_column in self.df.columns:
            value = self.df.at[index, self.display_column]
            if pd.notna(value):
                return str(value)
        return ""

    def _map_algorithm(self, algo: Optional[str], column: str) -> str:
        candidate = (algo or "").strip().lower()
        lowered = column.lower()
        if candidate in {"domain", "website", "url"}:
            return "domain"
        if candidate in {"email", "email_domain"}:
            return "email_domain"
        if candidate in {
            "token_set",
            "token_set_ratio",
            "token_sort_ratio",
            "wratio",
            "auto",
        }:
            return "token_set"
        if candidate in {"jaro", "jarowinkler", "jaro_winkler"}:
            return "jaro"
        if candidate == "contains":
            return "contains"
        if candidate == "exact":
            return "exact"
        if any(keyword in lowered for keyword in ("domain", "website", "url", "site")):
            return "domain"
        if "email" in lowered:
            return "email_domain"
        if any(
            keyword in lowered
            for keyword in ("name", "company", "account", "organization")
        ):
            return "token_set"
        if pd.api.types.is_numeric_dtype(self.df[column]):
            return "exact"
        return "jaro"

    def _mapping_options_for_algo(self, algo: str) -> MappingOptions:
        if algo in {"token_set", "name", "fuzzy"}:
            return MappingOptions(casefold=True, strip_punctuation=True)
        if algo in {"jaro", "contains"}:
            return MappingOptions(casefold=True, strip_punctuation=True)
        return MappingOptions()

    def _append_mapping(
        self,
        collection: List[MappingConfig],
        seen: set,
        source: str,
        reference: str,
        algo: str,
        weight: Any,
        opts: MappingOptions,
    ) -> None:
        if source not in self.df.columns or reference not in self.df.columns:
            return
        key = (source, reference, algo)
        if key in seen:
            return
        normalized_weight = self._normalize_weight(weight)
        collection.append(
            MappingConfig(
                source=source,
                reference=reference,
                algo=algo,
                weight=normalized_weight,
                opts=opts,
            )
        )
        seen.add(key)

    def _build_mappings(self) -> List[MappingConfig]:
        mappings: List[MappingConfig] = []
        seen: set = set()
        raw_mappings = self.config.get("mappings") or []
        for candidate in raw_mappings:
            source = candidate.get("source")
            if not source or source not in self.df.columns:
                continue
            reference = candidate.get("reference") or candidate.get("ref") or source
            if reference not in self.df.columns:
                reference = source
            algo = self._map_algorithm(
                candidate.get("algorithm") or candidate.get("preferred_algo"), source
            )
            weight = candidate.get("weight") or candidate.get("weight_pct") or 1.0
            opts = self._mapping_options_for_algo(algo)
            self._append_mapping(mappings, seen, source, reference, algo, weight, opts)
        for column in self.email_columns:
            self._append_mapping(
                mappings, seen, column, column, "email_domain", 1.0, MappingOptions()
            )
        for column in self.domain_columns:
            self._append_mapping(
                mappings, seen, column, column, "domain", 1.0, MappingOptions()
            )
        for column in self.name_columns:
            self._append_mapping(
                mappings,
                seen,
                column,
                column,
                "token_set",
                0.8,
                MappingOptions(casefold=True, strip_punctuation=True),
            )
        for column in self.state_columns:
            self._append_mapping(
                mappings,
                seen,
                column,
                column,
                "exact",
                0.5,
                MappingOptions(casefold=True),
            )
        for column in self.country_columns:
            self._append_mapping(
                mappings,
                seen,
                column,
                column,
                "exact",
                0.5,
                MappingOptions(casefold=True),
            )
        if not mappings:
            fallback_columns = [
                col
                for col in self.df.columns
                if col not in {"__row_id__"} and col != self.id_column
            ]
            if fallback_columns:
                column = fallback_columns[0]
                self._append_mapping(
                    mappings,
                    seen,
                    column,
                    column,
                    "token_set",
                    1.0,
                    MappingOptions(casefold=True, strip_punctuation=True),
                )
        self.mappings_used = mappings
        return mappings

    def _execute_manual_match(self) -> Tuple[pd.DataFrame, Dict[str, Any]]:
        mappings = self._build_mappings()
        blocks = (
            {"domain": True}
            if any(m.algo in {"domain", "email_domain"} for m in mappings)
            else None
        )
        anchors = (
            self.config.get("anchors")
            if isinstance(self.config.get("anchors"), dict)
            else None
        )
        domain_registry = None
        try:
            domain_registry = get_registry()
            logger.info(
                "[SHEETS] Domain registry initialized: %s families",
                domain_registry.to_dict().get("total_families"),
            )
        except Exception as exc:
            logger.warning(f"[SHEETS] Could not initialize domain registry: {exc}")
        result_df, metrics = manual_match_execute(
            source_df=self.df,
            reference_df=self.df,
            mappings=mappings,
            anchors=anchors,
            blocks=blocks,
            threshold=self.threshold,
            best_only=False,
            strip_b2c=True,
            domain_registry=domain_registry,
        )
        return result_df, metrics

    def _build_pair_results(self, result_df: pd.DataFrame) -> List[Dict[str, Any]]:
        pair_map: Dict[Tuple[int, int], Dict[str, Any]] = {}
        for _, row in result_df.iterrows():
            ref_idx = row.get("reference_row_index")
            if ref_idx in (None, "", "None"):
                continue
            try:
                ref_idx = int(ref_idx)
            except (TypeError, ValueError):
                continue
            source_idx = row.get("source_row_index")
            if source_idx in (None, "", "None"):
                source_idx = row.get("source_id")
                try:
                    source_idx = int(source_idx) - 1
                except (TypeError, ValueError):
                    continue
            try:
                source_idx = int(source_idx)
            except (TypeError, ValueError):
                continue
            if source_idx == ref_idx or source_idx < 0 or ref_idx < 0:
                continue
            match_status = str(row.get("match_status") or "").strip() or "Account Match"
            try:
                match_score = float(row.get("match_score") or 0)
            except (TypeError, ValueError):
                match_score = 0.0
            reasons = [
                token for token in str(row.get("reasons") or "").split("|") if token
            ]
            anchor_hits = [token for token in reasons if token.startswith("ANCHOR:")]
            strong_domain = any(
                reason in {"M:DOMAIN", "M:EMAIL_DOMAIN"} for reason in reasons
            )
            if (
                match_status != "Account Match"
                and match_score < self.threshold_pct
                and not strong_domain
            ):
                continue
            matched_domain = row.get("matched_domain") or ""
            explanation_raw = row.get("explanation") or "{}"
            try:
                explanation = json.loads(explanation_raw)
            except Exception:
                explanation = {}
            info = {
                "source_idx": source_idx,
                "target_idx": ref_idx,
                "match_score": match_score,
                "match_status": match_status,
                "reasons": reasons,
                "anchor_hits": anchor_hits,
                "matched_domain": matched_domain,
                "explanation": explanation,
                "candidate_rank": int(row.get("candidate_rank") or 1),
                "source_row_id": self._resolve_row_id(source_idx),
                "target_row_id": self._resolve_row_id(ref_idx),
            }
            key = tuple(sorted((source_idx, ref_idx)))
            existing = pair_map.get(key)
            if existing is None or match_score > existing["match_score"]:
                pair_map[key] = info
        if not pair_map:
            pair_map.update(self._build_domain_fallback_pairs())
        return list(pair_map.values())

    def _build_domain_fallback_pairs(self) -> Dict[Tuple[int, int], Dict[str, Any]]:
        """
        Build conservative duplicate candidates from shared domains.

        This keeps Sheets dedupe useful when the primary matcher only returns
        self-matches for very small datasets.
        """
        domain_to_rows: Dict[Tuple[str, str], set[int]] = defaultdict(set)

        for idx, row in self.df.iterrows():
            for column in self.email_columns:
                domain, _ = _extract_email_domain(row.get(column), strip_b2c=True)
                if domain:
                    domain_to_rows[("email", domain)].add(int(idx))
            for column in self.domain_columns:
                domain, _ = _extract_domain(row.get(column), strip_b2c=True)
                if domain:
                    domain_to_rows[("domain", domain)].add(int(idx))

        pair_map: Dict[Tuple[int, int], Dict[str, Any]] = {}
        for (kind, domain), rows in domain_to_rows.items():
            sorted_rows = sorted(rows)
            if len(sorted_rows) < 2:
                continue
            reason = "M:EMAIL_DOMAIN" if kind == "email" else "M:DOMAIN"
            for i in range(len(sorted_rows) - 1):
                for j in range(i + 1, len(sorted_rows)):
                    source_idx = sorted_rows[i]
                    target_idx = sorted_rows[j]
                    key = (source_idx, target_idx)
                    pair_map[key] = {
                        "source_idx": source_idx,
                        "target_idx": target_idx,
                        "match_score": max(self.threshold_pct, 90.0),
                        "match_status": "Account Match",
                        "reasons": [reason],
                        "anchor_hits": [],
                        "matched_domain": domain,
                        "explanation": {"fallback": "shared_domain"},
                        "candidate_rank": 1,
                        "source_row_id": self._resolve_row_id(source_idx),
                        "target_row_id": self._resolve_row_id(target_idx),
                    }
        return pair_map

    def _format_matches(self, pairs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        matches: List[Dict[str, Any]] = []
        for info in sorted(pairs, key=lambda item: item["match_score"], reverse=True):
            record_id = self._resolve_record_id(info["source_idx"])
            duplicate_of = self._resolve_record_id(info["target_idx"])
            match_details = {
                "match_score": info["match_score"],
                "match_status": info["match_status"],
                "reasons": info["reasons"],
                "anchor_hits": info["anchor_hits"],
                "matched_domain": info["matched_domain"],
                "explanation": info["explanation"],
                "candidate_rank": info["candidate_rank"],
                "threshold_pct": self.threshold_pct,
            }
            matches.append(
                {
                    "record_id": record_id,
                    "duplicate_of": duplicate_of,
                    "confidence": info["match_score"] / 100.0
                    if info["match_score"]
                    else 0.0,
                    "match_status": info["match_status"],
                    "reasons": info["reasons"],
                    "anchor_hits": info["anchor_hits"],
                    "matched_domain": info["matched_domain"],
                    "match_details": match_details,
                    "__row_id__": info["source_row_id"],
                    "__duplicate_row_id__": info["target_row_id"],
                }
            )
        return matches

    def _build_clusters(self, pairs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        if not pairs:
            return []
        graph: Dict[int, set] = defaultdict(set)
        pair_lookup: Dict[Tuple[int, int], Dict[str, Any]] = {}
        for info in pairs:
            a = info["source_idx"]
            b = info["target_idx"]
            graph[a].add(b)
            graph[b].add(a)
            key = tuple(sorted((a, b)))
            current = pair_lookup.get(key)
            if current is None or info["match_score"] > current["match_score"]:
                pair_lookup[key] = info
        visited = set()
        clusters: List[Dict[str, Any]] = []
        cluster_seq = 1
        for node in sorted(graph.keys()):
            if node in visited:
                continue
            component = set()
            stack = [node]
            while stack:
                current = stack.pop()
                if current in visited:
                    continue
                visited.add(current)
                component.add(current)
                for neighbor in graph[current]:
                    if neighbor not in visited:
                        stack.append(neighbor)
            if len(component) < 2:
                continue
            records = []
            for idx in sorted(component):
                records.append(
                    {
                        "__row_id__": self._resolve_row_id(idx),
                        "record_id": self._resolve_record_id(idx),
                        "display_name": self._resolve_display_value(idx),
                    }
                )
            links = []
            for key, info in pair_lookup.items():
                a, b = key
                if a in component and b in component:
                    links.append(
                        {
                            "record_id": self._resolve_record_id(info["source_idx"]),
                            "duplicate_of": self._resolve_record_id(info["target_idx"]),
                            "confidence": info["match_score"] / 100.0
                            if info["match_score"]
                            else 0.0,
                            "match_status": info["match_status"],
                            "reasons": info["reasons"],
                            "anchor_hits": info["anchor_hits"],
                            "matched_domain": info["matched_domain"],
                        }
                    )
            clusters.append(
                {
                    "cluster_id": f"C{cluster_seq:04d}",
                    "records": records,
                    "links": links,
                }
            )
            cluster_seq += 1
        return clusters

    def _compute_domain_frequencies(self) -> Dict[str, Counter]:
        email_freq: Counter[str] = Counter()
        for column in self.email_columns:
            series = self.df[column].dropna()
            for value in series:
                domain, _ = _extract_email_domain(value, strip_b2c=True)
                if domain:
                    email_freq[domain] += 1
        website_freq: Counter[str] = Counter()
        for column in self.domain_columns:
            series = self.df[column].dropna()
            for value in series:
                domain, _ = _extract_domain(value, strip_b2c=True)
                if domain:
                    website_freq[domain] += 1
        return {"email": email_freq, "website": website_freq}

    def _build_telemetry(
        self,
        metrics: Dict[str, Any],
        pairs: List[Dict[str, Any]],
        clusters: List[Dict[str, Any]],
    ) -> Dict[str, Any]:
        domain_freq = self._compute_domain_frequencies()
        anchor_counter = Counter()
        reason_counter = Counter()
        for info in pairs:
            anchor_counter.update(info["anchor_hits"])
            reason_counter.update(info["reasons"])
        return {
            "metrics": metrics,
            "domain_frequency": {
                "email": [
                    {"domain": domain, "count": count}
                    for domain, count in domain_freq["email"].most_common(20)
                ],
                "website": [
                    {"domain": domain, "count": count}
                    for domain, count in domain_freq["website"].most_common(20)
                ],
            },
            "anchor_hits": [
                {"anchor": anchor, "count": count}
                for anchor, count in anchor_counter.items()
                if anchor
            ],
            "reason_counts": [
                {"reason": reason, "count": count}
                for reason, count in reason_counter.items()
            ],
            "cluster_count": len(clusters),
            "threshold_pct": self.threshold_pct,
            "b2c_stripped_count": metrics.get("b2c_stripped_count", 0),
            "mappings_used": [
                {
                    "source": m.source,
                    "reference": m.reference,
                    "algo": m.algo,
                    "weight": m.weight,
                }
                for m in self.mappings_used
            ],
            "anchors": self.config.get("anchors"),
            "blocks": "domain"
            if any(m.algo in {"domain", "email_domain"} for m in self.mappings_used)
            else "none",
        }

    def _build_stats(
        self,
        metrics: Dict[str, Any],
        matches: List[Dict[str, Any]],
        clusters: List[Dict[str, Any]],
    ) -> Dict[str, Any]:
        return {
            "total_records": int(len(self.df)),
            "duplicate_groups": len(clusters),
            "duplicates_found": len(matches),
            "matches_by_status": metrics.get("matches_by_status", {}),
            "avg_candidates_per_row": metrics.get("avg_candidates_per_row"),
            "score_distribution": metrics.get("score_distribution", {}),
            "top_reasons": metrics.get("top_reasons", []),
            "blocked_rows": metrics.get("blocked_rows", 0),
            "blocked_zero_but_has_alt_domain": metrics.get(
                "blocked_zero_but_has_alt_domain", 0
            ),
            "b2c_stripped_count": metrics.get("b2c_stripped_count", 0),
            "domain_normalized_count": metrics.get("domain_normalized_count", 0),
            "names_normalized_count": metrics.get("names_normalized_count", 0),
            "domain_overlap_upper_bound": metrics.get("domain_overlap_upper_bound", 0),
        }

    def run(self) -> Dict[str, Any]:
        result_df, metrics = self._execute_manual_match()
        pairs = self._build_pair_results(result_df)
        matches = self._format_matches(pairs)
        clusters = self._build_clusters(pairs)
        telemetry = self._build_telemetry(metrics, pairs, clusters)
        stats = self._build_stats(metrics, matches, clusters)
        return {
            "matches": matches,
            "duplicates": clusters,
            "duplicate_groups": len(clusters),
            "duplicates_found": len(matches),
            "stats": stats,
            "telemetry": telemetry,
        }


# Inline processing functions for small datasets
async def process_dedupe_inline(
    df: pd.DataFrame, config: Dict[str, Any]
) -> Dict[str, Any]:
    """Process dedupe synchronously for small datasets using manual match parity."""
    runner = SheetsManualDedupe(df, config or {})
    return runner.run()


async def process_match_inline(
    source_df: pd.DataFrame, reference_df: pd.DataFrame, config: Dict[str, Any]
) -> Dict[str, Any]:
    """Process match synchronously for small datasets"""
    from ...core.matcher import FuzzyMatcher

    # Initialize matcher
    matcher = FuzzyMatcher(mode="match")

    # Configure matcher
    if config.get("mappings"):
        for mapping in config["mappings"]:
            matcher.add_mapping(
                mapping.get("source"),
                mapping.get("reference"),
                weight=mapping.get("weight", 1.0),
                algorithm=mapping.get("algorithm", "WRatio"),
            )

    # Set threshold
    matcher.set_threshold(config.get("threshold", 80))

    # Run match
    results = matcher.match(
        source_df,
        reference_df,
        source_id=config.get("source_id_column"),
        reference_id=config.get("reference_id_column"),
    )

    # Format results
    matches = []
    source_id_field = config.get("source_id_column") or "source_id"
    reference_id_field = config.get("reference_id_column") or "reference_id"

    for _, row in results.iterrows():
        matched_ref = (
            row.get("matched_id")
            or row.get(reference_id_field)
            or row.get("reference_id")
        )
        if not matched_ref:
            continue

        source_identifier = row.get(source_id_field) or row.get("source_id")

        matches.append(
            {
                "source_id": str(source_identifier)
                if source_identifier is not None
                else None,
                "reference_id": str(matched_ref),
                "confidence": row.get("confidence", row.get("score", 0)),
                "match_details": row.get("match_details", {}),
            }
        )

    return {"matches": matches}


# Background processing functions for large datasets
async def process_sheets_dedupe_job(
    job_id: UUID,
    df: pd.DataFrame,
    config: Dict[str, Any],
    account_id: Optional[str],
    db: AsyncSession,
):
    """Process dedupe job in background"""
    try:
        job_repo = JobRepository(db)

        # Update job status
        await job_repo.update_job_status(str(job_id), status=JobStatus.PROCESSING.value)

        # Process dedupe
        results = await process_dedupe_inline(df, config)
        summary = results.get("stats") or {
            "duplicate_groups": results.get("duplicate_groups"),
            "duplicates_found": results.get("duplicates_found"),
        }

        # Update job with results
        await job_repo.update_job_status(
            str(job_id),
            status=JobStatus.COMPLETED.value,
            results={
                "preview": results.get("matches", [])[:100],
                "summary": summary,
            },
            matches_found=len(results.get("matches", [])),
        )
        if account_id:
            await record_quota_usage(
                account_id, rows_returned=len(results.get("matches", [])), db=db
            )

    except Exception as e:
        logger.error(f"Error processing sheets dedupe job {job_id}: {str(e)}")
        await job_repo.update_job_status(
            str(job_id), status=JobStatus.FAILED.value, error=str(e)
        )


async def process_sheets_match_job(
    job_id: UUID,
    source_df: pd.DataFrame,
    reference_df: pd.DataFrame,
    config: Dict[str, Any],
    account_id: Optional[str],
    db: AsyncSession,
):
    """Process match job in background"""
    try:
        job_repo = JobRepository(db)

        # Update job status
        await job_repo.update_job_status(str(job_id), status=JobStatus.PROCESSING.value)

        # Process match
        results = await process_match_inline(source_df, reference_df, config)

        # Update job with results
        await job_repo.update_job_status(
            str(job_id),
            status=JobStatus.COMPLETED.value,
            results={
                "preview": results.get("matches", [])[:100],
                "summary": {"matches_found": len(results.get("matches", []))},
            },
            matches_found=len(results.get("matches", [])),
        )
        if account_id:
            await record_quota_usage(
                account_id, rows_returned=len(results.get("matches", [])), db=db
            )

    except Exception as e:
        logger.error(f"Error processing sheets match job {job_id}: {str(e)}")
        await job_repo.update_job_status(
            str(job_id), status=JobStatus.FAILED.value, error=str(e)
        )
