"""Text deduplication utilities for finding and removing duplicate texts."""

from typing import Dict, List, Tuple, Union, Optional, Any

from freamon.deduplication.exact_deduplication import (
    hash_deduplication,
    ngram_fingerprint_deduplication
)

from freamon.deduplication.fuzzy_deduplication import (
    find_similar_texts,
    deduplicate_texts,
    calculate_levenshtein_similarity,
    calculate_jaccard_similarity
)

from freamon.deduplication.fingerprinting import (
    create_shingled_document,
    create_minhash_signature,
    create_simhash_signature
)

from freamon.deduplication.clustering import (
    cluster_texts_hierarchical,
    cluster_texts_dbscan
)

from freamon.deduplication.lsh_deduplication import (
    lsh_deduplication,
    create_lsh_bands
)

from freamon.deduplication.supervised_deduplication import (
    SupervisedDeduplicationModel
)

from freamon.deduplication.report import (
    generate_deduplication_report,
    export_deduplication_report,
    prepare_duplicate_report_data
)

# Import duplicate flagging functionality
from freamon.deduplication.flag_duplicates import (
    flag_exact_duplicates,
    flag_text_duplicates,
    flag_similar_records,
    flag_supervised_duplicates,
    add_duplicate_detection_columns
)

# Import Polars-optimized implementations
from freamon.deduplication.polars_lsh_deduplication import (
    polars_lsh_deduplication,
    streaming_lsh_deduplication,
    batch_process_texts,
    batch_create_minhash_signatures
)

from freamon.deduplication.polars_supervised_deduplication import (
    PolarsSupervisedDeduplicationModel
)

__all__ = [
    # Exact deduplication
    'hash_deduplication',
    'ngram_fingerprint_deduplication',
    
    # Fuzzy deduplication
    'find_similar_texts',
    'deduplicate_texts',
    'calculate_levenshtein_similarity',
    'calculate_jaccard_similarity',
    
    # Document fingerprinting
    'create_shingled_document',
    'create_minhash_signature',
    'create_simhash_signature',
    
    # Clustering
    'cluster_texts_hierarchical',
    'cluster_texts_dbscan',
    
    # LSH deduplication
    'lsh_deduplication',
    'create_lsh_bands',
    
    # Supervised deduplication
    'SupervisedDeduplicationModel',
    
    # Deduplication reporting
    'generate_deduplication_report',
    'export_deduplication_report',
    'prepare_duplicate_report_data',
    
    # Duplicate flagging functionality
    'flag_exact_duplicates',
    'flag_text_duplicates',
    'flag_similar_records',
    'flag_supervised_duplicates',
    'add_duplicate_detection_columns',
    
    # Polars-optimized LSH deduplication
    'polars_lsh_deduplication',
    'streaming_lsh_deduplication',
    'batch_process_texts',
    'batch_create_minhash_signatures',
    
    # Polars-optimized supervised deduplication
    'PolarsSupervisedDeduplicationModel'
]
