"""Deduplication utilities for ingestion.

These functions require the 'ingest' extra dependencies:
    pip install fts-mcp[ingest]
"""

from tqdm.auto import tqdm


def deduplicate_exact(records: list[dict], deduplicate_by: str) -> list[dict]:
    """Remove exact duplicate records based on normalized field values.

    Args:
        records: List of record dicts
        deduplicate_by: Field name to use for deduplication

    Returns:
        Deduplicated list of records
    """
    from lenlp import normalizer

    seen = set()
    deduplicated = []
    for record in tqdm(records):
        key = normalizer.normalize(record[deduplicate_by])
        if key not in seen:
            deduplicated.append(record)
            seen.add(key)
    print(
        f"Deduplication removed {len(records) - len(deduplicated)} of {len(records)} records."
    )
    return deduplicated


def deduplicate_fuzzy(
    records: list[dict], deduplicate_by: str, cutoff: float = 95
) -> list[dict]:
    """Remove fuzzy duplicate records based on similarity threshold.

    Args:
        records: List of record dicts
        deduplicate_by: Field name to use for deduplication
        cutoff: Similarity threshold (0-100), records above this are considered duplicates

    Returns:
        Deduplicated list of records
    """
    from lenlp import normalizer
    from rapidfuzz import fuzz, process

    result = []
    normalized_seen = []

    for record in records:
        key = normalizer.normalize(record[deduplicate_by])
        if normalized_seen:
            max_sim = process.extract(
                query=key,
                choices=normalized_seen,
                scorer=fuzz.ratio,
            )[0][1]
            if max_sim > cutoff:
                continue
        result.append(record)
        normalized_seen.append(key)

    print(
        f"Deduplication removed {len(records) - len(result)} of {len(records)} records."
    )
    return result
