"""
import pandas as pd
import numpy as np
import sklearn
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
from sklearn.neighbors import NearestNeighbors
from sklearn.ensemble import IsolationForest
from statsmodels.tsa.stattools import acf

def get_dynamic_lags(series: pd.Series) -> list:
    
    n = len(series)
    
    # Determine Max Lags (Max is min(50% of data, a hard cap of 60))
    nlags = min(int(n * 0.5), 60)
    
    if nlags < 5:
        return [1, 2, 3]

    # Calculate ACF and Confidence Intervals, get the 10 most-significant lags
    autocorrelations, confint = acf(series.dropna(), nlags=nlags, alpha=0.25, fft=True)
    autocorr_values = autocorrelations[1:]
    conf_limit = confint[1:, 1] - autocorr_values
    is_significant = np.abs(autocorr_values) > conf_limit
    significant_autocorr = autocorr_values[is_significant]
    significant_lags_indices = np.where(is_significant)[0] + 1
    ranked_indices = np.argsort(np.abs(significant_autocorr))[::-1]
    top_lags_indices = ranked_indices[:10]
    top_lags = significant_lags_indices[top_lags_indices].tolist()
    base_lags = [1, 2, 3]
    dynamic_lags = sorted(list(set(base_lags + top_lags)))[:10]
    
    return dynamic_lags

def find_optimal_epsilon(X_scaled: np.ndarray, k: int) -> float:

    Finds the optimal epsilon by calculating the distance to the k-th nearest neighbor
    and taking a high percentile (90-95th) of those distances as the cutoff.

    if len(X_scaled) < k:
        return 1.0 # Fallback 

    # Find the distance to the k-th (min_samples) neighbor for every point
    # n_neighbors is k+1 because the first distance is 0 (to itself)
    neigh = NearestNeighbors(n_neighbors=k + 1) 
    neigh.fit(X_scaled)
    
    # distances matrix: [n_samples, k+1]
    distances, indices = neigh.kneighbors(X_scaled)
    
    # We are interested in the distance to the k-th neighbor (index k)
    k_distances = distances[:, k] 
    
    # The elbow is hard to find programmatically. A robust proxy for the density
    # threshold is to take a high percentile (e.g., 95th) of the k-distances. 
    optimal_eps = np.percentile(k_distances, 95) 
    
    # --- ADJUSTMENT: Loosen up by 20% ---
    # Reducing epsilon makes the neighborhood smaller, meaning fewer points 
    # qualify as core points, resulting in MORE anomalies (approx 20% looser).
    optimal_eps = optimal_eps * 0.80
    
    # Ensure a minimum value if data is extremely sparse
    return max(optimal_eps, 0.1)

def calculate_scaled_score(k_distance, epsilon):

    Scales the anomaly score to be between 0 and 1.
    - Normals (distance <= epsilon): Scaled 0 to 0.49 (Linear)
    - Anomalies (distance > epsilon): Scaled 0.51 to 1.0 (Exponential S-Curve)

    # Calculate ratio of distance to the threshold
    # Avoid division by zero
    ratio = k_distance / (epsilon + 1e-10)
    
    if ratio <= 1.0:
        # Normal Range: Map [0, 1] -> [0, 0.49]
        # We use linear scaling here so points closer to the boundary get higher scores
        return ratio * 0.49
    else:
        # Anomaly Range: Map (1, inf) -> [0.51, 1.0]
        # We use an exponential saturation curve: 1 - exp(-(ratio-1))
        # If ratio = 1 (boundary), exp(0) = 1 -> 1-1 = 0 -> Score 0.51
        # If ratio = 2 (double distance), exp(-1) = 0.36 -> 1-0.36 = 0.64 -> Score ~0.82
        # If ratio = 5 (extreme), exp(-4) ~ 0 -> Score ~ 1.0
        # This ensures distinct separation for severe anomalies
        return 0.51 + 0.49 * (1 - np.exp(-(ratio - 1)))

def detect_time_series_anomalies_dbscan(
    group,
    variable,
    date_column,
    eval_period,
    ):
    

    # 🌀 DBSCAN Walk-Forward Anomaly Detection (Scaled 0-1 Score)


    # Get copy of the original group df
    group_orig = group.copy()
    
    # Keep only relevant columns for analysis, which are variable and date_column
    # group_orig may have columns that have NaNs or NoneTypes and those rows will be removed in a .dropna() function below
    group = group[[variable, date_column]]
    
    group[date_column] = pd.to_datetime(group[date_column])
    group = group.copy().sort_values(date_column).reset_index(drop=True)
    group['set'] = np.where(np.arange(len(group)) >= len(group) - eval_period, 'TEST', 'TRAIN')
    
    try:
        all_results = []

        # ===================================================================
        # STEP 1: Evaluate all points in the initial TRAIN period
        # ===================================================================
        
        # Prepare the full group with features
        model_group_initial = group.copy()
        
        # Get train set to determine lags
        train_initial = model_group_initial[model_group_initial['set'] == 'TRAIN'].copy()
        lags = get_dynamic_lags(train_initial[variable])

        # Create lag features and rolling stats for the entire DF
        rolling_stats_features = []
        for lag in lags:
            model_group_initial[f'lag{lag}'] = model_group_initial[variable].shift(lag)

        for w in [int(np.ceil(max(lags)/4)), int(np.ceil(max(lags)/2)), int(max(lags))]:
            if w >= 3:
                rolling_stats_features.extend([f'roll_mean_{w}', f'roll_std_{w}'])
                model_group_initial[f'roll_mean_{w}'] = model_group_initial[variable].shift(1).rolling(w).mean()
                model_group_initial[f'roll_std_{w}'] = model_group_initial[variable].shift(1).rolling(w).std()

        model_group_initial['trend'] = model_group_initial.index
        model_group_initial = model_group_initial.copy().dropna()

        # Get just the initial train set
        train_initial = model_group_initial[model_group_initial['set'] == 'TRAIN'].copy()

        # Identify all model features (lags, rolling stats, trend, and the variable itself)
        features = [f'lag{i}' for i in lags] + rolling_stats_features + ['trend'] + [variable]

        # Fit the scaler on the training data
        scaler = StandardScaler()
        scaler.fit(train_initial[features])
        train_scaled = scaler.transform(train_initial[features])

        # Determine min_samples based on feature space dimension
        min_samples = max(2 * len(features), 3)
        
        # Find optimal epsilon
        calculated_eps = find_optimal_epsilon(train_scaled, k=min_samples)

        # --- DBSCAN MODEL on initial training data ---
        dbscan_model = DBSCAN(
            eps=calculated_eps, 
            min_samples=min_samples,
            n_jobs=-1
        )

        # Fit DBSCAN on the scaled training data
        cluster_labels = dbscan_model.fit_predict(train_scaled)
        
        # For training points, use DBSCAN labels (-1 = noise/anomaly)
        train_initial['is_DBSCAN_anomaly'] = (cluster_labels == -1)
        
        # Calculate raw distances for scoring
        neigh = NearestNeighbors(n_neighbors=min_samples)
        neigh.fit(train_scaled)
        distances, indices = neigh.kneighbors(train_scaled)
        # Store raw distance and epsilon for scaling later
        train_initial['k_distance'] = distances[:, min_samples - 1]
        train_initial['epsilon_threshold'] = calculated_eps
        
        train_initial['DBSCAN_anomaly'] = np.where(train_initial['is_DBSCAN_anomaly'] == True, 'high', 'none')
        
        # Select columns to keep for now
        train_initial_result = train_initial[[variable, date_column, 'k_distance', 'epsilon_threshold', 'is_DBSCAN_anomaly', 'DBSCAN_anomaly']]
        
        all_results.append(train_initial_result)

        # ===================================================================
        # STEP 2: Walk-forward evaluation for TEST period (one-step-ahead)
        # ===================================================================
        
        for t in list(range(eval_period - 1, -1, -1)):

            try:
                # Boundary between rolling train and rolling forecast region
                cutoff_date = group[date_column].max() - pd.Timedelta(weeks=t)

                # Get train set to determine lags
                model_group = group.copy()
                train = model_group[model_group[date_column] <= cutoff_date].copy()
                lags = get_dynamic_lags(train[variable])

                # Create lag features and rolling stats for the entire DF
                rolling_stats_features = []
                for lag in lags:
                    model_group[f'lag{lag}'] = model_group[variable].shift(lag)

                for w in [int(np.ceil(max(lags)/4)), int(np.ceil(max(lags)/2)), int(max(lags))]:
                    if w >= 3:
                        rolling_stats_features.extend([f'roll_mean_{w}', f'roll_std_{w}'])
                        model_group[f'roll_mean_{w}'] = model_group[variable].shift(1).rolling(w).mean()
                        model_group[f'roll_std_{w}'] = model_group[variable].shift(1).rolling(w).std()

                model_group['trend'] = group.index
                model_group = model_group.copy().dropna()

                # Split into train and test
                train = model_group[model_group[date_column] <= cutoff_date].copy()
                test = model_group[model_group[date_column] == cutoff_date].copy()

                # Identify all model features (lags, rolling stats, trend, and the variable itself)
                features = [f'lag{i}' for i in lags] + rolling_stats_features + ['trend'] + [variable]

                # Fit the scaler ONLY on the training data to avoid data leakage
                scaler = StandardScaler()
                scaler.fit(train[features])
                
                # Transform both train and test sets
                train_scaled = scaler.transform(train[features])
                test_scaled = scaler.transform(test[features])

                # Determine min_samples based on feature space dimension
                min_samples = max(2 * len(features), 3)
                
                # Find optimal epsilon
                calculated_eps = find_optimal_epsilon(train_scaled, k=min_samples)

                # --- NOVELTY DETECTION ---
                neigh = NearestNeighbors(n_neighbors=min_samples)
                neigh.fit(train_scaled)
                
                # Find the distance of the test point to its nearest neighbors in the train set
                distances, indices = neigh.kneighbors(test_scaled)
                
                # Use the distance to the k-th neighbor (index min_samples-1)
                k_distance = distances[:, min_samples - 1] 
                
                # Flag as anomaly if the k-distance is greater than the trained eps threshold
                test['k_distance'] = k_distance
                test['epsilon_threshold'] = calculated_eps
                test['is_DBSCAN_anomaly'] = np.where(k_distance > calculated_eps, True, False)
                test['DBSCAN_anomaly'] = np.where(test['is_DBSCAN_anomaly'] == True, 'high', 'none')
                
                test = test[[variable, date_column, 'k_distance', 'epsilon_threshold', 'is_DBSCAN_anomaly', 'DBSCAN_anomaly']]
                all_results.append(test)

            except Exception as e:
                print(f"Error in iteration {t}: {e}")
                pass
            
        # ===================================================================
        # STEP 3: Combine all results, Scale Scores, and merge back
        # ===================================================================
        
        try:
            all_results_df = pd.concat(all_results, ignore_index=True)
            
            # --- APPLY SCALING LOGIC HERE ---
            # We vectorize the scaling function for performance
            v_scale_score = np.vectorize(calculate_scaled_score)
            
            all_results_df['DBSCAN_score'] = v_scale_score(
                all_results_df['k_distance'], 
                all_results_df['epsilon_threshold']
            )
            
            # Create the 'high' flag (just visual consistency with legacy code)
            all_results_df['DBSCAN_score_high'] = 0
            
            # Merge back to original group
            group = group.merge(
                all_results_df[[variable, date_column, 'DBSCAN_score', 
                               'DBSCAN_score_high', 'is_DBSCAN_anomaly', 'DBSCAN_anomaly']], 
                on=[variable, date_column], 
                how='left'
            )
            
        except Exception as e:
            print(f"Error in concatenating results: {e}")
            group['DBSCAN_score'] = np.nan
            group['DBSCAN_score_high'] = np.nan
            group["is_DBSCAN_anomaly"] = np.nan
            group["DBSCAN_anomaly"] = np.nan

    except Exception as e:
        # Fallback error handling
        try:
             group_id_cols = group.select_dtypes(include=['object', 'string']).columns.tolist()
             group_id = " ".join(group[group_id_cols].reset_index(drop=True).iloc[0].astype(str).to_list())
        except:
             group_id = "Unknown Group ID"
        print(f'DBSCAN Anomaly Detection failed for {group_id}. Error: {e}')
        group['DBSCAN_score'] = np.nan
        group['DBSCAN_score_high'] = np.nan
        group["is_DBSCAN_anomaly"] = np.nan
        group["DBSCAN_anomaly"] = np.nan
    
    group = pd.concat([group_orig, group.drop(columns=[variable, date_column])], axis=1)
    
    return group 
    """
import pandas as pd
import numpy as np
from sklearn.neighbors import NearestNeighbors
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
from statsmodels.tsa.stattools import acf

def get_dynamic_lags(series: pd.Series) -> list:
    n = len(series)
    nlags = min(int(n * 0.5), 60)
    if nlags < 5: return [1, 2, 3]
    autocorrelations, confint = acf(series.dropna(), nlags=nlags, alpha=0.25, fft=True)
    autocorr_values = autocorrelations[1:]
    conf_limit = confint[1:, 1] - autocorr_values
    is_significant = np.abs(autocorr_values) > conf_limit
    sig_indices = np.where(is_significant)[0] + 1
    ranked = np.argsort(np.abs(autocorr_values[is_significant]))[::-1]
    top_lags = sig_indices[ranked[:5]].tolist() 
    return sorted(list(set([1, 2, 3] + top_lags)))[:7]

def find_optimal_epsilon(X_scaled: np.ndarray, k: int, sensitivity: float) -> float:
    if len(X_scaled) <= k: return 0.5
    neigh = NearestNeighbors(n_neighbors=k).fit(X_scaled)
    distances, _ = neigh.kneighbors(X_scaled)
    # TIGHTENED: Changed from 90th to 80th percentile to shrink the "normal" bubble
    return max(np.percentile(distances[:, -1], 80) * sensitivity, 0.1)

def calculate_scaled_score(k_distance, epsilon):
    ratio = k_distance / (epsilon + 1e-10)
    if ratio <= 1.0:
        return ratio * 0.49
    else:
        return 0.51 + 0.49 * (1 - np.exp(-1.5 * (ratio - 1)))

def detect_time_series_anomalies_dbscan(
    group, 
    variable, 
    date_column, 
    eval_period, 
    sensitivity=0.95 # Nudged down to increase detection rate
):
    
    group = group.reset_index(drop=True)
    group_orig = group.copy()
    df_proc = group[[variable, date_column]].copy()
    df_proc[date_column] = pd.to_datetime(df_proc[date_column])
    df_proc = df_proc.sort_values(date_column).reset_index(drop=True)
    
    try:
        all_results = []
        train_cutoff_idx = len(df_proc) - eval_period
        
        # --- BALANCED NOISE REDUCTION ---
        # 5-day window is the 'sweet spot' for 3% detection
        df_proc['smoothed_val'] = df_proc[variable].rolling(window=5, min_periods=1).mean()
        df_proc['velocity'] = df_proc['smoothed_val'].diff()
        
        features = ['smoothed_val', 'velocity']
        
        # Min samples set to 6 for better local sensitivity
        min_samples = 6 

        def find_eps_balanced(X, k, sens):
            if len(X) <= k: return 0.5
            neigh = NearestNeighbors(n_neighbors=k).fit(X)
            dist, _ = neigh.kneighbors(X)
            # 92nd percentile targets the 3% detection range effectively
            return max(np.percentile(dist[:, -1], 92) * sens, 0.2)

        # --- STEP 1: TRAIN ---
        test_start_date = df_proc.iloc[train_cutoff_idx][date_column]
        train_df = df_proc[df_proc[date_column] < test_start_date].dropna().copy()
        
        scaler = StandardScaler()
        train_sc = scaler.fit_transform(train_df[features])
        eps = find_eps_balanced(train_sc, min_samples, sensitivity)
        
        dbscan = DBSCAN(eps=eps, min_samples=min_samples).fit(train_sc)
        
        train_df['k_distance'] = eps 
        train_df['epsilon_threshold'] = eps
        train_df['is_DBSCAN_anomaly'] = (dbscan.labels_ == -1)
        all_results.append(train_df)

        # --- STEP 2: WALK FORWARD ---
        model_df = df_proc.dropna().reset_index(drop=True)
        test_indices = model_df[model_df[date_column] >= test_start_date].index
        for idx in test_indices:
            # Look back 80 points for a stable but reactive baseline
            curr_train = model_df.iloc[max(0, idx-80):idx].copy()
            curr_test = model_df.iloc[[idx]].copy()
            
            sc = StandardScaler()
            tr_sc = sc.fit_transform(curr_train[features])
            te_sc = sc.transform(curr_test[features])
            
            local_eps = find_eps_balanced(tr_sc, min_samples, sensitivity)
            nn_test = NearestNeighbors(n_neighbors=min_samples).fit(tr_sc)
            d, _ = nn_test.kneighbors(te_sc)
            
            curr_test['k_distance'] = d[0][-1]
            curr_test['epsilon_threshold'] = local_eps
            curr_test['is_DBSCAN_anomaly'] = d[0][-1] > local_eps
            all_results.append(curr_test)

        # --- STEP 3: CONSOLIDATE ---
        res_df = pd.concat(all_results).drop_duplicates(subset=[date_column])
        v_scale = np.vectorize(calculate_scaled_score)
        res_df['DBSCAN_score'] = v_scale(res_df['k_distance'], res_df['epsilon_threshold'])
        
        res_df['is_DBSCAN_anomaly'] = res_df['is_DBSCAN_anomaly'].astype(bool)
        res_df['DBSCAN_anomaly'] = np.where(res_df['is_DBSCAN_anomaly'], 'high', 'none')
        res_df['DBSCAN_score_high'] = res_df['is_DBSCAN_anomaly'].astype(int)

        group_orig[date_column] = pd.to_datetime(group_orig[date_column])
        final = group_orig.merge(
            res_df[[date_column, 'DBSCAN_score', 'DBSCAN_score_high', 'is_DBSCAN_anomaly', 'DBSCAN_anomaly']], 
            on=date_column, how='left'
        )
        
        final['DBSCAN_score'] = final['DBSCAN_score'].fillna(0)
        final['DBSCAN_score_high'] = final['DBSCAN_score_high'].fillna(0).astype(int)
        final['is_DBSCAN_anomaly'] = final['is_DBSCAN_anomaly'].fillna(False).astype(bool)
        final['DBSCAN_anomaly'] = final['DBSCAN_anomaly'].fillna('none')
        
        return final

    except Exception as e:
        print(f"Error in DBSCAN: {e}")
        for col in ["DBSCAN_score", "DBSCAN_score_high", "is_DBSCAN_anomaly", "DBSCAN_anomaly"]:
            group_orig[col] = 0
        return group_orig