import pandas as pd
import numpy as np
import sklearn
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
from sklearn.neighbors import NearestNeighbors
from sklearn.ensemble import IsolationForest
from statsmodels.tsa.stattools import acf


def get_dynamic_lags(series: pd.Series) -> list:
    
    n = len(series)
    
    # Determine Max Lags (Max is min(50% of data, a hard cap of 60))
    nlags = min(int(n * 0.5), 60)
    
    if nlags < 5:
        return [1, 2, 3]

    # Calculate ACF and Confidence Intervals, get the 10 most-significant lags
    autocorrelations, confint = acf(series.dropna(), nlags=nlags, alpha=0.25, fft=True)
    autocorr_values = autocorrelations[1:]
    conf_limit = confint[1:, 1] - autocorr_values
    is_significant = np.abs(autocorr_values) > conf_limit
    significant_autocorr = autocorr_values[is_significant]
    significant_lags_indices = np.where(is_significant)[0] + 1
    ranked_indices = np.argsort(np.abs(significant_autocorr))[::-1]
    top_lags_indices = ranked_indices[:10]
    top_lags = significant_lags_indices[top_lags_indices].tolist()
    base_lags = [1, 2, 3]
    dynamic_lags = sorted(list(set(base_lags + top_lags)))[:10]
    
    return dynamic_lags


def find_optimal_epsilon(X_scaled: np.ndarray, k: int) -> float:
    """
    Finds the optimal epsilon by calculating the distance to the k-th nearest neighbor
    and taking a high percentile (90-95th) of those distances as the cutoff.
    This serves as a programmatic proxy for the 'elbow' method in a rolling window.
    """
    if len(X_scaled) < k:
        return 1.0 # Fallback 

    # Find the distance to the k-th (min_samples) neighbor for every point
    # n_neighbors is k+1 because the first distance is 0 (to itself)
    neigh = NearestNeighbors(n_neighbors=k + 1) 
    neigh.fit(X_scaled)
    
    # distances matrix: [n_samples, k+1]
    distances, indices = neigh.kneighbors(X_scaled)
    
    # We are interested in the distance to the k-th neighbor (index k)
    # This k-distance is the required radius for a point to be a core point's neighbor.
    k_distances = distances[:, k] 
    
    # The elbow is hard to find programmatically. A robust proxy for the density
    # threshold is to take a high percentile (e.g., 95th) of the k-distances. 
    # This sets epsilon such that 95% of your *training* points would be considered
    # part of a cluster's neighborhood.
    optimal_eps = np.percentile(k_distances, 95) 
    
    # Ensure a minimum value if data is extremely sparse
    return max(optimal_eps, 0.1)


def detect_time_series_anomalies_dbscan(
    group,
    variable,
    date_column,
    eval_period,
    ):
    
    """# 🌀 DBSCAN Walk-Forward Anomaly Detection
    ---

    The `detect_time_series_anomalies_dbscan` function implements a **density-based clustering** approach for time-series anomaly detection. It utilizes an **iterative walk-forward validation** strategy to identify data points that exist in "low-density" regions of the feature space.

    ## 📋 Functional Overview
    This function transforms a univariate time series into a high-dimensional feature space using **dynamic lags** and **rolling statistics**. It then applies the **DBSCAN** (Density-Based Spatial Clustering of Applications with Noise) algorithm to distinguish between dense clusters of "normal" behavior and sparse "noise" points (anomalies).

    ## 🧠 Core Logic & Helper Utilities

    ### 1. Dynamic Feature Engineering (`get_dynamic_lags`)
    Instead of using fixed lags, the function uses the **Autocorrelation Function (ACF)** to find the 10 most significant seasonal patterns in the data.
    * **Baseline:** Always includes lags 1, 2, and 3 to capture immediate momentum.
    * **Significance:** Uses a 75% confidence interval ($\\\\alpha=0.25$) to identify meaningful historical dependencies.

    ### 2. Automated Parameter Tuning (`find_optimal_epsilon`)
    DBSCAN is highly sensitive to the **Epsilon ($\\\\epsilon$)** parameter (the neighborhood radius). 
    * **Proxy Elbow Method:** The function automatically calculates $\\\\epsilon$ by analyzing the distance to the $k$-th nearest neighbor for all training points.
    * **Density Threshold:** It sets $\\\\epsilon$ at the **95th percentile** of these distances, ensuring that 95% of training data is considered "dense" while the most isolated 5% are candidates for noise.

    ### 3. Walk-Forward Iteration
    For the initial training period, all points are evaluated using DBSCAN fitted on the same training data.
    For each period in the `eval_period`:
    * **Feature Construction:** Builds a matrix containing the variable, its dynamic lags, rolling means, rolling standard deviations, and a linear trend component.
    * **Scaling:** Fits a `StandardScaler` **only on training data** to prevent data leakage.
    * **Novelty Detection:** Since DBSCAN cannot "predict" on new points, the function uses a **Nearest Neighbors proxy**. If the distance from a new test point to its $k$-th neighbor in the training set is greater than the trained $\\\\epsilon$, it is flagged as an anomaly.

    ## 📤 Key Output Columns
    * **`dbscan_score`**: The distance from the point to the $\\\\epsilon$ boundary (positive values indicate anomalies).
    * **`is_DBSCAN_anomaly`**: A boolean flag identifying outliers.
    * **Generated Features**: Includes all dynamic lags (`lagX`) and rolling statistics (`roll_mean_W`) used during the fit.

    ## 💡 Usage Context
    DBSCAN is exceptionally powerful for detecting **contextual anomalies**—points that might look "normal" in value but are "weird" given their recent history or seasonal context. Because it is density-based, it can find anomalies in non-linear or multi-modal distributions where simple percentile or Z-score methods would fail.

    ---
    ### ⚠️ Performance Note
    This model is computationally more intensive than statistical methods due to the iterative re-fitting of the `NearestNeighbors` and `DBSCAN` models. It is best suited for high-priority metrics where accuracy is more critical than processing speed."""

    group[date_column] = pd.to_datetime(group[date_column])
    group = group.copy().sort_values(date_column).reset_index(drop=True)
    group['set'] = np.where(np.arange(len(group)) >= len(group) - eval_period, 'TEST', 'TRAIN')
    
    # --- Default DBSCAN Parameters ---
    # These parameters often need tuning, but these are reasonable starting points:
    DEFAULT_EPS = 0.5 # Neighborhood radius (critical parameter)
    
    try:
        all_results = []

        # ===================================================================
        # STEP 1: Evaluate all points in the initial TRAIN period
        # ===================================================================
        
        # Get the cutoff date for initial train period
        initial_cutoff_date = group[group['set'] == 'TRAIN'][date_column].max()
        
        # Prepare the full group with features
        model_group_initial = group.copy()
        
        # Get train set to determine lags
        train_initial = model_group_initial[model_group_initial['set'] == 'TRAIN'].copy()
        lags = get_dynamic_lags(train_initial[variable])

        # Create lag features and rolling stats for the entire DF
        rolling_stats_features = []
        for lag in lags:
            model_group_initial[f'lag{lag}'] = model_group_initial[variable].shift(lag)

        for w in [int(np.ceil(max(lags)/4)), int(np.ceil(max(lags)/2)), int(max(lags))]:
            if w >= 3:
                rolling_stats_features.extend([f'roll_mean_{w}', f'roll_std_{w}'])
                model_group_initial[f'roll_mean_{w}'] = model_group_initial[variable].shift(1).rolling(w).mean()
                model_group_initial[f'roll_std_{w}'] = model_group_initial[variable].shift(1).rolling(w).std()

        model_group_initial['trend'] = model_group_initial.index
        model_group_initial = model_group_initial.copy().dropna()

        # Get just the initial train set
        train_initial = model_group_initial[model_group_initial['set'] == 'TRAIN'].copy()

        # Identify all model features (lags, rolling stats, trend, and the variable itself)
        features = [f'lag{i}' for i in lags] + rolling_stats_features + ['trend'] + [variable]

        # Fit the scaler on the training data
        scaler = StandardScaler()
        scaler.fit(train_initial[features])
        train_scaled = scaler.transform(train_initial[features])

        # Determine min_samples based on feature space dimension
        min_samples = max(2 * len(features), 3)
        
        # Find optimal epsilon
        calculated_eps = find_optimal_epsilon(train_scaled, k=min_samples)

        # --- DBSCAN MODEL on initial training data ---
        dbscan_model = DBSCAN(
            eps=calculated_eps, 
            min_samples=min_samples,
            n_jobs=-1
        )

        # Fit DBSCAN on the scaled training data
        cluster_labels = dbscan_model.fit_predict(train_scaled)
        
        # For training points, use DBSCAN labels (-1 = noise/anomaly)
        train_initial['is_DBSCAN_anomaly'] = (cluster_labels == -1)
        
        # Calculate scores for training points
        # Use distance to k-th nearest neighbor as score
        neigh = NearestNeighbors(n_neighbors=min_samples)
        neigh.fit(train_scaled)
        distances, indices = neigh.kneighbors(train_scaled)
        k_distance = distances[:, min_samples - 1]
        
        train_initial['dbscan_score'] = k_distance - calculated_eps
        train_initial['dbscan_score_high'] = 0
        
        # Select relevant columns
        train_initial_result = train_initial[[variable, date_column, 'dbscan_score', 
                                             'dbscan_score_high', 'is_DBSCAN_anomaly']]
        all_results.append(train_initial_result)

        # ===================================================================
        # STEP 2: Walk-forward evaluation for TEST period (one-step-ahead)
        # ===================================================================
        
        for t in list(range(eval_period - 1, -1, -1)):

            try:
                # Boundary between rolling train and rolling forecast region
                cutoff_date = group[date_column].max() - pd.Timedelta(weeks=t)

                # Get train set to determine lags
                model_group = group.copy()
                train = model_group[model_group[date_column] <= cutoff_date].copy()
                lags = get_dynamic_lags(train[variable])

                # Create lag features and rolling stats for the entire DF
                rolling_stats_features = []
                for lag in lags:
                    model_group[f'lag{lag}'] = model_group[variable].shift(lag)

                for w in [int(np.ceil(max(lags)/4)), int(np.ceil(max(lags)/2)), int(max(lags))]:
                    if w >= 3:
                        rolling_stats_features.extend([f'roll_mean_{w}', f'roll_std_{w}'])
                        model_group[f'roll_mean_{w}'] = model_group[variable].shift(1).rolling(w).mean()
                        model_group[f'roll_std_{w}'] = model_group[variable].shift(1).rolling(w).std()

                model_group['trend'] = group.index
                model_group = model_group.copy().dropna()

                # Split into train and test
                train = model_group[model_group[date_column] <= cutoff_date].copy()
                test = model_group[model_group[date_column] == cutoff_date].copy()

                # Identify all model features (lags, rolling stats, trend, and the variable itself)
                features = [f'lag{i}' for i in lags] + rolling_stats_features + ['trend'] + [variable]

                # Fit the scaler ONLY on the training data to avoid data leakage
                scaler = StandardScaler()
                
                # Fit the scaler on the train data features
                scaler.fit(train[features])
                
                # Transform both train and test sets
                train_scaled = scaler.transform(train[features])
                test_scaled = scaler.transform(test[features])

                # Determine min_samples based on feature space dimension
                min_samples = max(2 * len(features), 3)
                
                # Find optimal epsilon
                calculated_eps = find_optimal_epsilon(train_scaled, k=min_samples)

                # --- DBSCAN MODEL ---
                dbscan_model = DBSCAN(
                    eps=calculated_eps, 
                    min_samples=min_samples,
                    n_jobs=-1
                )

                # Fit DBSCAN on the scaled training data
                dbscan_model.fit(train_scaled)

                # Since DBSCAN doesn't have a direct predict() method for new data points,
                # the simplest (and common) proxy is to treat the test point as unassigned noise,
                # which requires complex distance logic.
                
                neigh = NearestNeighbors(n_neighbors=min_samples)
                neigh.fit(train_scaled)
                
                # Find the distance of the test point to its nearest neighbors in the train set
                distances, indices = neigh.kneighbors(test_scaled)
                
                # Anomaly check: If the distance to the min_samples-th neighbor is > eps, it's noise.
                # Use the distance to the k-th neighbor (index min_samples-1)
                k_distance = distances[:, min_samples - 1] 
                
                # Flag as anomaly if the k-distance is greater than the trained eps threshold
                test['dbscan_score'] = k_distance - calculated_eps
                test['dbscan_score_high'] = 0
                test['is_DBSCAN_anomaly'] = np.where(test['dbscan_score'] > 0, True, False)
                
                test = test[[variable, date_column, 'dbscan_score', 'dbscan_score_high', 'is_DBSCAN_anomaly']]
                all_results.append(test)

            except Exception as e:
                print(f"Error in iteration {t}: {e}")
                pass
            
        # ===================================================================
        # STEP 3: Combine all results and merge back to original group
        # ===================================================================
        
        try:
            all_results_df = pd.concat(all_results, ignore_index=True)
            
            # Merge back to original group
            group = group.merge(
                all_results_df[[variable, date_column, 'dbscan_score', 
                               'dbscan_score_high', 'is_DBSCAN_anomaly']], 
                on=[variable, date_column], 
                how='left'
            )
            
            # Fill any remaining NaNs with False for boolean column
            # group["is_DBSCAN_anomaly"] = group["is_DBSCAN_anomaly"].fillna(False)
            
        except Exception as e:
            print(f"Error in concatenating results: {e}")
            group['dbscan_score'] = np.nan
            group['dbscan_score_high'] = np.nan
            group["is_DBSCAN_anomaly"] = np.nan

    except Exception as e:
        # Fallback error handling
        try:
             group_id_cols = group.select_dtypes(include=['object', 'string']).columns.tolist()
             group_id = " ".join(group[group_id_cols].reset_index(drop=True).iloc[0].astype(str).to_list())
        except:
             group_id = "Unknown Group ID"
        print(f'DBSCAN Anomaly Detection failed for {group_id}. Error: {e}')
        group['dbscan_score'] = np.nan
        group['dbscan_score_high'] = np.nan
        group["is_DBSCAN_anomaly"] = np.nan
        
    return group