# data validation and optmization in ML pipelines

import hashlib
import json
from typing import Dict, Any
import pandas as pd
import numpy as np


# creating hash for a dataset to track changes and detect drift
def generate_data_fingerprint(df: pd.DataFrame, include_schema: bool = True) -> str:
    # Hash of shape + column names + dtypes
    schema_str = str(list(df.columns)) + str([str(d) for d in df.dtypes])

    # Sample hash (first and last 100 rows)
    sample = pd.concat([df.head(100), df.tail(100)])
    data_hash = hashlib.md5(sample.to_json().encode()).hexdigest()[:16]

    if include_schema:
        return f"{data_hash}_{hashlib.md5(schema_str.encode()).hexdigest()[:8]}"
    return data_hash


def estimate_memory_usage(df: pd.DataFrame, unit: str = "MB") -> float:
    """Estimate memory usage with unit conversion."""
    bytes_usage = df.memory_usage(deep=True).sum()

    units = {"B": 1, "KB": 1024, "MB": 1024**2, "GB": 1024**3}

    return bytes_usage / units.get(unit.upper(), 1024**2)


def detect_separator(file_path: str, sample_size: int = 4096) -> str:
    """Auto-detect CSV separator."""
    with open(file_path, "r") as f:
        sample = f.read(sample_size)

    separators = [",", ";", "\t", "|"]
    counts = {sep: sample.count(sep) for sep in separators}

    # return max(counts, key=counts.get)
    return str(max(counts.items(), key=lambda x: x[1])[0])


def optimize_dtypes(df: pd.DataFrame) -> pd.DataFrame:
    """
    Optimizing DataFrame memory usage by downcasting types.
    """
    optimized = df.copy()

    for col in optimized.columns:
        col_type = optimized[col].dtype

        if col_type != object:
            c_min = optimized[col].min()
            c_max = optimized[col].max()

            if str(col_type)[:3] == "int":
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    optimized[col] = optimized[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    optimized[col] = optimized[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    optimized[col] = optimized[col].astype(np.int32)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    optimized[col] = optimized[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    optimized[col] = optimized[col].astype(np.float32)
        else:
            # Convert to category if beneficial
            num_unique_values = len(optimized[col].unique())
            num_total_values = len(optimized[col])
            if num_unique_values / num_total_values < 0.5:
                optimized[col] = optimized[col].astype("category")

    return optimized
