"""
Duplicate detection and removal utilities.

Provides functions to find and remove exact duplicate rows,
which is a critical preprocessing step to prevent data leakage.
"""
import warnings
warnings.filterwarnings('ignore')
from dataclasses import dataclass
from typing import Optional, Sequence, Tuple, List, Set, Literal
import pandas as pd
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

from ..utils.validation import (
    validate_dataframe,
    validate_columns_exist,
    validate_non_empty_dataframe,
    validate_keep_parameter,
    validate_ratio,
)

#요약
@dataclass(frozen=True)
class ExactDuplicateSummary:
    """
    Summary statistics for exact duplicate detection.

    Attributes
    ----------
    n_rows : int
        Total number of rows in original dataset.
    exact_duplicate_rows : int
        Number of duplicate rows found.
    exact_duplicate_percentage : float
        Percentage of duplicate rows (0-100).
    """
    n_rows: int
    exact_duplicate_rows: int
    exact_duplicate_percentage: float

    def __str__(self) -> str:
        return (
            f"ExactDuplicateSummary("
            f"n_rows={self.n_rows}, "
            f"duplicates={self.exact_duplicate_rows}, "
            f"percentage={self.exact_duplicate_percentage:.2f}%"
            f")"
        )


@dataclass(frozen=True)
class NearDuplicateSummary:
    """
    Summary statistics for near-duplicate detection.

    Attributes
    ----------
    n_rows : int
        Total number of rows in original dataset.
    near_duplicate_pairs : int
        Number of near-duplicate pairs found.
    affected_rows : int
        Total number of rows involved in near-duplicate pairs.
    affected_percentage : float
        Percentage of affected rows (0-100).
    similarity_threshold : float
        Threshold used for detection (0.0-1.0).
    """
    n_rows: int
    near_duplicate_pairs: int
    affected_rows: int
    affected_percentage: float
    similarity_threshold: float

    def __str__(self) -> str:
        """
        Return concise string representation for debugging.
        """
        return (
            f"NearDuplicateSummary("
            f"n_rows={self.n_rows}, "
            f"pairs={self.near_duplicate_pairs}, "
            f"affected_rows={self.affected_rows}, "
            f"affected_pct={self.affected_percentage:.2f}%, "
            f"threshold={self.similarity_threshold:.2f}"
            f")"
        )


def find_exact_duplicates(
    df: pd.DataFrame,
    subset: Optional[Sequence[str]] = None,
    keep: Literal["first", "last", False] = "first",
) -> Tuple[pd.DataFrame, ExactDuplicateSummary]:
    """
    find exact duplicate rows.

    Parameters
    ----------
    df : pd.DataFrame
        Input dataset.
    subset : Optional[Sequence[str]]
        Columns to consider. If None, use all columns.
    keep : str, default="first"
        Which duplicates to mark:
        - 'first': Mark duplicates except for the first occurrence
        - 'last': Mark duplicates except for the last occurrence
        - False: Mark all duplicates (including first occurrence)

    Returns
    -------
    duplicates : pd.DataFrame
        DataFrame containing only the duplicate rows.
    summary : ExactDuplicateSummary
        Statistics about duplicates found.

    Raises
    ------
    TypeError
        If df is not a pandas DataFrame or keep is invalid type.
    ValueError
        If keep parameter is invalid or subset columns don't exist.

    Examples
    --------
    >>> df = pd.DataFrame({
    ...     'Date': ['2024-01-01', '2024-01-01', '2024-01-02'],
    ...     'Price': [100, 100, 200]
    ... })
    >>> dups, summary = find_exact_duplicates(df)
    >>> print(summary)
    Exact Duplicate Detection Summary
    Total rows: 3
    Duplicate rows: 1
    Duplicate percentage: 33.33%
    """
    validate_dataframe(df, name="df")
    validate_keep_parameter(keep, allow_false=True)
    validate_columns_exist(df, subset, name='df')

    # find exact duplicates
    dup_mask = df.duplicated(subset=subset, keep=keep)
    dup_df = df.loc[dup_mask].copy()

    # compute summary statistics
    n_rows = len(df)
    exact_dup_rows = int(dup_mask.sum())

    if n_rows == 0:
        pct = 0.0
    else:
        pct = (exact_dup_rows / n_rows) * 100.0

    summary = ExactDuplicateSummary(
        n_rows=n_rows,
        exact_duplicate_rows=exact_dup_rows,
        exact_duplicate_percentage=pct,
    )
    return dup_df, summary

def remove_exact_duplicates(
    df: pd.DataFrame,
    subset: Optional[Sequence[str]] = None,
    keep: Literal["first", "last", False] = "first",
) -> pd.DataFrame:
    """
    Remove exact duplicate rows.

    Parameters
    ----------
    df : pd.DataFrame
        Input dataset.
    subset : Sequence[str], optional
        Columns to consider for duplicate detection.
        If None, uses all columns.
    keep : str, default='first'
        Which duplicate to keep:
        - 'first': Keep first occurrence, remove others
        - 'last': Keep last occurrence, remove others

    Returns
    -------
    pd.DataFrame
        DataFrame with duplicates removed and index reset.

    Raises
    ------
    TypeError
        If df is not a pandas DataFrame or keep is invalid type.
    ValueError
        If keep parameter is invalid or subset columns don't exist.

    Examples
    --------
    >>> df = pd.DataFrame({
    ...     'A': [1, 1, 2],
    ...     'B': [10, 10, 20]
    ... })
    >>> clean = remove_exact_duplicates(df)
    >>> len(clean)
    2
    """
    validate_dataframe(df, name="df")
    validate_keep_parameter(keep, allow_false=False)
    validate_columns_exist(df, subset, name="df")

    return df.drop_duplicates(subset=subset, keep=keep).reset_index(drop=True)

def _extract_and_convert_numeric(
        df: pd.DataFrame,
        subset: Optional[Sequence[str]] = None
) -> tuple[pd.DataFrame, List[str]]:
    """
    Internal helper: Extract numeric columns with format conversion.

    This is a preprocessing step to use cosine similarity.
    Converts formatted strings (e.g., "1,000", "1.5M") to numbers.
    """
    df_work = df.copy()  # Don't modify original

    if subset is None:
        candidate_cols = df_work.columns.tolist()
    else:
        candidate_cols = list(subset)

    numeric_cols = []

    for col in candidate_cols:
        series = df_work[col]

        if pd.api.types.is_object_dtype(series) or pd.api.types.is_string_dtype(series):
            series = series.str.replace(",", "", regex=False)
            series = (
                series
                .str.replace("K", "e3", regex=False)
                .str.replace("M", "e6", regex=False)
                .str.replace("B", "e9", regex=False)
            )

        converted = pd.to_numeric(series, errors="coerce")

        if converted.notna().sum() > 0:
            df_work[col] = converted
            numeric_cols.append(col)

    return df_work, numeric_cols

def find_near_duplicates(
    df: pd.DataFrame,
    subset: Optional[Sequence[str]] = None,
    threshold: float = 0.95,
    sample_size: int = None,
    max_rows: int = 10_000,
    random_state: int = 42,
) -> Tuple[List[Tuple[int, int]], NearDuplicateSummary, pd.DataFrame]:
    """
    find near-duplicate rows using configurable similarity threshold.

    Purpose
    -------
    Identifies rows that are highly similar but not exactly identical.
    Near-duplicates can also cause data leakage if the same or very
    similar samples appear in both training and test sets.

    Algorithm
    ---------
    1. Extract numeric features from specified columns
    2. Normalize features to [0, 1] range
    3. Compute pairwise cosine similarity
    4. Identify pairs exceeding the similarity threshold

    Parameters
    ----------
    df : pd.DataFrame
        Input dataset.
    subset : Optional[Sequence[str]], default=None
        Numeric columns to consider for similarity calculation.
        If None, uses all numeric columns.
    threshold : float, default=0.95
        Similarity threshold between 0.0 and 1.0.
        - 1.0: Only exact matches (same as exact duplicates)
        - 0.95: Rows must be 95% similar or more
        - 0.90: Rows must be 90% similar or more
        Lower values find more pairs but may include false positives.
    sample_size : int, optional
        Number of observations to sample from each split.
        If None, automatic sampling is applied when needed.
    max_rows : int, default=10_000
        Maximum number of train-test pairs before triggering sampling.
        Prevents memory overflow on large datasets.
    random_state : int, default=42
        Random seed for reproducible sampling.

    Returns
    -------
    pairs : List[Tuple[int, int]]
        List of (index_1, index_2) pairs that are near-duplicates.
        Indices refer to positions in the sampled dataset (df_sampled).
        Use df_sampled.iloc[idx] to access actual rows.
    summary : NearDuplicateSummary
        Statistics about near-duplicates found.
    df_used : pd.DataFrame
        The dataframe actually used for analysis.
        - If sampling occurred: sampled subset
        - If no sampling: original df (by reference, not copy)

    Raises
    ------
    TypeError
        If df is not a DataFrame or threshold is not numeric.
    ValueError
        If df is empty, threshold is not in [0,1], or no numeric columns found.

    Notes
    -----
    - Computational complexity: O(n²) where n is number of rows
    - For large datasets (>10,000 rows), automatic sampling is applied
    - Missing values are filled with column means before computation
    - Uses cosine similarity, which works well for normalized numeric data
    - IMPORTANT: The returned df_sampled should be used in downstream analysis
      (e.g., measure_leakage) to ensure consistency across pipeline steps
    - Pair indices are relative to df_sampled, not the original df

    Examples
    --------
    >>> df = pd.DataFrame({
    ...     'price': [100.0, 100.5, 200.0],
    ...     'volume': [50, 51, 100]
    ... })
    >>> pairs, summary, df_sample = find_near_duplicates(df, threshold=0.99)
    >>> print(summary)
    Near-Duplicate Detection Summary
    Total rows: 3
    Similarity threshold: 0.99
    Near-duplicate pairs found: 1
    Affected rows: 2

    >>> # To see which specific rows are similar:
    >>> for idx1, idx2 in pairs:
    ...     print(f"Row {idx1} ~ Row {idx2}")
    Row 0 ~ Row 1
    """
    # Validate inputs
    validate_dataframe(df, name="df")
    validate_non_empty_dataframe(df, name="df")
    validate_ratio(threshold, name="threshold", inclusive=True)

    # Check if sampling is needed
    needs_sampling = len(df) > max_rows

    if needs_sampling and sample_size is None:
        # Auto-calculate sample size
        sample_size = max_rows
        warnings.warn(
            f"Large dataset detected ({len(df):,} rows). "
            f"Applying automatic sampling (sample_size={sample_size:,}). "
            f"Near-duplicate detection will be performed on sample.",
            UserWarning
        )

    # Apply sampling if needed
    if sample_size is not None and len(df) > sample_size:
        np.random.seed(random_state)
        df_used = df.sample(n=sample_size, random_state=random_state)
        print(f"Sampling applied: analyzing {len(df_used):,} of {len(df):,} rows")

    else:
        df_used = df.copy()  # Always return a copy to prevent modifying original

    # Select numeric columns for similarity calculation
    df_numeric = df_used.copy()
    df_numeric, numeric_cols = _extract_and_convert_numeric(df_numeric, subset)

    if len(numeric_cols) == 0:
        raise ValueError(
            f"No numeric columns found for similarity calculation. "
        )

    # Extract numeric data
    data = df_numeric[numeric_cols].values.astype(float)
    n_rows = len(df_numeric)

    # Handle missing values (fill with column mean)
    if np.isnan(data).any():
        col_means = np.nanmean(data, axis=0)
        for i, col_mean in enumerate(col_means):
            mask = np.isnan(data[:, i]) # i번째 컬럼에서 NaN 위치 찾기
            data[mask, i] = col_mean # NaN을 평균값으로 채우기

    # Normalize each column to [0, 1] range
    data_min = data.min(axis=0)
    data_max = data.max(axis=0)
    data_range = data_max - data_min # 최대-최소

    # Avoid division by zero for constant columns
    # 더 안전한 처리
    data_range = np.where(data_range == 0, 1.0, data_range)  # 0 처리
    data_range = np.where(np.isnan(data_range), 1.0, data_range)  # NaN 처리
    data_normalized = (data - data_min) / data_range
    data_normalized = np.nan_to_num(data_normalized, nan=0.0)  # 남은 NaN 처리

    # Compute pairwise cosine similarity(use sklearn)
    similarity_matrix = cosine_similarity(data_normalized)

    # Find pairs above threshold (excluding diagonal and duplicates)
    pairs = []
    for i in range(n_rows):
        for j in range(i + 1, n_rows):
            if similarity_matrix[i, j] >= threshold:
                pairs.append((i, j))

    # Count affected rows (unique indices involved in any pair)
    affected_indices: Set[int] = set() # set을 쓰면 중복 자동 제거

    for idx1, idx2 in pairs:
        affected_indices.add(idx1)
        affected_indices.add(idx2)

    affected_indices_list = sorted(list(affected_indices))

    affected_rows = len(affected_indices_list)
    affected_pct = (affected_rows / n_rows * 100.0) if n_rows > 0 else 0.0

    summary = NearDuplicateSummary(
        n_rows=n_rows,
        near_duplicate_pairs=len(pairs),
        affected_rows=affected_rows,
        affected_percentage=affected_pct,
        similarity_threshold=threshold,
    )

    return pairs, summary, df_used

def remove_near_duplicates(
    df: pd.DataFrame,
    subset: Optional[Sequence[str]] = None,
    threshold: float = 0.95,
    keep: str = "first",
) -> pd.DataFrame:
    """
    Remove near-duplicate rows based on similarity threshold.

    Purpose
    -------
    Optional removal step for near-duplicates, controlled by user parameter.
    For each pair of near-duplicates, keeps only one occurrence based on
    the 'keep' parameter.

    Workflow
    --------
    1. Find all near-duplicate pairs using find_near_duplicates()
    2. For each pair, mark one row for removal based on 'keep' parameter
    3. Remove marked rows and reset index

    Parameters
    ----------
    df : pd.DataFrame
        Input dataset.
    subset : Optional[Sequence[str]], default=None
        Numeric columns to consider for similarity.
        If None, uses all numeric columns.
    threshold : float, default=0.95
        Similarity threshold (0.0-1.0).
        See find_near_duplicates() for details.
    keep : str, default="first"
        Which duplicate to keep when a pair is found:
        - 'first': Keep the row with lower index
        - 'last': Keep the row with higher index

    Returns
    -------
    pd.DataFrame
        DataFrame with near-duplicates removed and index reset.

    Raises
    ------
    TypeError
        If df is not a DataFrame or keep is invalid type.
    ValueError
        If keep is not 'first' or 'last', or other validation errors.

    Notes
    -----
    - If a row is similar to multiple other rows, the removal strategy
      may not be deterministic. Consider reviewing pairs before removal.
    - For complex deduplication logic, consider using find_near_duplicates()
      and implementing custom removal logic.

    Examples
    --------
    >>> df = pd.DataFrame({
    ...     'price': [100.0, 100.1, 200.0],
    ...     'volume': [50, 50, 100]
    ... })
    >>> clean = remove_near_duplicates(df, threshold=0.99, keep='first')
    >>> len(clean)  # Second row removed as it's very similar to first
    2
    """
    validate_dataframe(df, name="df")
    validate_keep_parameter(keep, allow_false=False)

    # Find near-duplicate pairs
    pairs, _, _ = find_near_duplicates(df, subset=subset, threshold=threshold)

    # If no near-duplicates found, return copy of original
    if len(pairs) == 0:
        return df.copy()

    # Collect indices to remove
    indices_to_remove: Set[int] = set()

    for idx1, idx2 in pairs:
        if keep == "first":
            # Keep the one with lower index (first occurrence)
            indices_to_remove.add(idx2)
        else:  # keep == "last"
            # Keep the one with higher index (last occurrence)
            indices_to_remove.add(idx1)

    # Remove marked rows and reset index
    mask = ~df.index.isin(indices_to_remove)
    return df[mask].reset_index(drop=True)

def find_all_duplicates(
    df: pd.DataFrame,
    subset: Optional[Sequence[str]] = None,
    near_dup_threshold: float = 0.95,
    print_summary: bool = True,
) -> Tuple[ExactDuplicateSummary, NearDuplicateSummary]:
    """
    Find both exact and near duplicates in one call.

    Purpose
    -------
    Convenience function to run both duplicate detection methods
    and optionally print a combined summary report.

    Parameters
    ----------
    df : pd.DataFrame
        Input dataset.
    subset : Optional[Sequence[str]], default=None
        Columns to consider. If None:
        - Exact duplicates: uses all columns
        - Near duplicates: uses all numeric columns
    near_dup_threshold : float, default=0.95
        Similarity threshold for near-duplicate detection.
    print_summary : bool, default=True
        Whether to print summary reports to console.

    Returns
    -------
    exact_summary : ExactDuplicateSummary
        Summary of exact duplicate detection.
    near_summary : NearDuplicateSummary
        Summary of near-duplicate detection.

    Examples
    --------
    >>> df = pd.DataFrame({
    ...     'Date': ['2024-01-01', '2024-01-01', '2024-01-02'],
    ...     'Price': [100, 100.1, 200]
    ... })
    >>> exact_sum, near_sum = find_all_duplicates(df, near_dup_threshold=0.99)

    Exact Duplicate Detection Summary
    Total rows: 3
    Duplicate rows: 1
    ...

    Near-Duplicate Detection Summary
    Total rows: 3
    Near-duplicate pairs found: 1
    ...
    """
    validate_dataframe(df, name="df")

    # Find exact duplicates
    _, exact_summary = find_exact_duplicates(df, subset=subset)

    # Find near duplicates (only on numeric columns)
    try:
        _, near_summary, _ = find_near_duplicates(
            df,
            subset=subset,
            threshold=near_dup_threshold
        )
    except ValueError as e:
        # If no numeric columns, create empty summary
        if "No numeric columns" in str(e):
            near_summary = NearDuplicateSummary(
                n_rows=len(df),
                near_duplicate_pairs=0,
                affected_rows=0,
                affected_percentage=0.0,
                similarity_threshold=near_dup_threshold,
            )
        else:
            raise

    # Print summaries if requested
    if print_summary:
        print(exact_summary)
        print(near_summary)

    return exact_summary, near_summary