import pandas as pd
import numpy as np
from datetime import datetime

def classify(val,lower,upper):
    if val < lower:
        return 'low'
    elif val > upper:
        return 'high'
    else:
        return 'none'

"""
def create_full_calendar_and_interpolate(
        master_data,
        group_columns,
        variable,
        date_column,
        freq,
        min_records,
        max_records
    ):
    master_data[date_column] = pd.to_datetime(master_data[date_column])
    
    full_group_data = []
    success_metrics = []
    dropped_metrics = []

    for group_key, group in master_data.groupby(group_columns):
        # Create a dictionary of the group keys for structured reporting
        # This maps {col1: val1, col2: val2}
        current_group_info = {
            col: group_key[i] if isinstance(group_key, (tuple, list)) else group_key 
            for i, col in enumerate(group_columns)
        }
        
        # 1. Calendar Generation
        min_date, max_date = group[date_column].min(), group[date_column].max()
        full_dates = pd.date_range(start=min_date, end=max_date, freq=freq)
        
        if max_records is not None and len(full_dates) > max_records:
            full_dates = full_dates[-max_records:]

        # 2. Expansion
        calendar_dict = current_group_info.copy()
        calendar_dict[date_column] = full_dates
        full_calendar = pd.DataFrame(calendar_dict)

        # 3. Merge
        merged = full_calendar.merge(group, on=group_columns + [date_column], how="left")
        
        total_len = len(merged)
        interpolated_count = merged[variable].isna().sum()
        interpolation_rate = interpolated_count / total_len if total_len > 0 else 0

        # --- Check 1: Min Records ---
        if total_len < min_records:
            drop_entry = current_group_info.copy()
            drop_entry.update({
                "reason": "Below Min Records",
                "details": f"Total records {total_len} < {min_records}",
                "dropped_records": total_len
            })
            dropped_metrics.append(drop_entry)
            continue

        # --- Check 2: Max Interpolation Rate ---
        if interpolation_rate > 0.25:
            drop_entry = current_group_info.copy()
            drop_entry.update({
                "reason": "High Interpolation",
                "details": f"{interpolation_rate:.1%} > 25%",
                "dropped_records": total_len
            })
            dropped_metrics.append(drop_entry)
            continue

        # --- Success: Interpolate ---
        merged["is_missing_record"] = merged[variable].isna()
        merged[variable] = merged[variable].interpolate(method="linear", limit_direction="both")

        success_entry = current_group_info.copy()
        success_entry.update({
            "initial_records": len(group),
            "interpolated_count": interpolated_count,
            "final_records": total_len,
            "interpolation_pct": round(interpolation_rate * 100, 2)
        })
        success_metrics.append(success_entry)
        full_group_data.append(merged)

    # Convert lists of dicts to DataFrames
    final_df = pd.concat(full_group_data, ignore_index=True) if full_group_data else pd.DataFrame()
    success_report = pd.DataFrame(success_metrics)
    exclusion_report = pd.DataFrame(dropped_metrics)
    
    return final_df, success_report, exclusion_report


def print_anomaly_stats(final_results, success_report, exclusion_report,group_columns,interpolation_method="linear"):
    # 1. Calculate Global Counts
    total_records = len(final_results)
    total_anomalies = final_results['is_Anomaly'].fillna(False).astype(bool).sum()
    anomaly_rate = (total_anomalies / total_records) * 100 if total_records > 0 else 0

    # 2. Extract specific stats from reports
    # If exclusion_report is passed but empty, len() returns 0
    num_excluded = len(exclusion_report)
    total_groups = len(success_report) + num_excluded
    evaluated_groups = len(success_report)
    Evaluated_records = success_report['final_records'].sum() 
    
    # Interpolation stats
    total_interpolated_records = success_report['interpolated_count'].sum() if not success_report.empty else 0
    groups_with_interpolation = success_report[success_report['interpolated_count'] > 0].shape[0] if not success_report.empty else 0

    # 3. Handle Exclusion stats (check if empty to avoid filtering errors)
    if num_excluded > 0:
        missing_data_exclusions = exclusion_report[exclusion_report['reason'] == "High Interpolation"].shape[0]
        insufficient_history_exclusions = exclusion_report[exclusion_report['reason'] == "Below Min Records"].shape[0]
    else:
        missing_data_exclusions = 0
        insufficient_history_exclusions = 0

    # --- START PRINTING ---
    print("\n" + "="*55)
    print(f"{'ANOMALY DETECTION EXECUTIVE SUMMARY':^55}")
    print("="*55)
    
    stats_table = [
        ["Total Groups", f"{total_groups:,}"],
        ["Total Records", f"{total_records:,}"],
        ["Evaluated Groups", f"{evaluated_groups:,}"],
        ["Evaluated Records", f"{Evaluated_records:,}"],
        ["Detected Anomalies", f"{total_anomalies:,}"],
        ["Anomaly Rate", f"{anomaly_rate:.2f}%"]
    ]
    
    for label, val in stats_table:
        print(f"{label:<25} : {val:>25}")
    
    print("-" * 55)

    # Interpolation Details
    print(f"INTERPOLATION REPORT:")
    print(f"{total_interpolated_records:,} records were missing from {groups_with_interpolation} groups")
    print(f"Values were interpolated using the {interpolation_method} method.")
    print(f"Total {total_interpolated_records:,} additional records are added to the data.")
    
    # Show 5 examples of interpolated records
    if 'is_interpolated' in final_results.columns:
        interpolated_samples = final_results[final_results['is_interpolated'] == True].head(5)
        if not interpolated_samples.empty:
            print("\nExample Interpolated Records:")
            # Only show group columns, timestamp (ds), and value (y)
            cols_to_show = group_columns + ['ds', 'y']
            print(interpolated_samples[cols_to_show].to_string(index=False))
    
    print("-" * 55)

    # Exclusion Details
    print(f"EXCLUSION SUMMARY:")
    if num_excluded > 0:
        print(f"- {missing_data_exclusions} groups had >25% missing data and could not be interpolated.")
        print(f"- {insufficient_history_exclusions} groups lacked the minimum historical data to train.")
        print(f"See exclusion_report for full list of IDs.")
    else:
        print("- No groups were excluded from this run.")

    print("-" * 55)
    
    # Group Breakdown
    print(f"TOP 5 GROUPS BY ANOMALY RATE ({' > '.join(group_columns)}):")
    group_stats = final_results.groupby(group_columns)['is_Anomaly'].agg(['mean', 'sum']).sort_values(by='mean', ascending=False).head(5)
    
    for label, row in group_stats.iterrows():
        # Handle tuple-based index for multi-grouping
        group_label = label if isinstance(label, (str, int)) else " | ".join(map(str, label))
        print(f" - {group_label:<30} : {row['mean']*100:>6.2f}% ({int(row['sum'])} anomalies)")
    
    print("="*55 + "\n") """

def create_full_calendar_and_interpolate(
        master_data,
        group_columns,
        variable,
        date_column,
        freq,
        min_records,
        max_records
    ):
    master_data[date_column] = pd.to_datetime(master_data[date_column])
    
    full_group_data = []
    success_metrics = []
    dropped_metrics = []

    for group_key, group in master_data.groupby(group_columns):
        # Capture the raw record count before any processing
        raw_record_count = len(group)
        
        current_group_info = {
            col: group_key[i] if isinstance(group_key, (tuple, list)) else group_key 
            for i, col in enumerate(group_columns)
        }
        
        # 1. Calendar Generation
        min_date, max_date = group[date_column].min(), group[date_column].max()
        full_dates = pd.date_range(start=min_date, end=max_date, freq=freq)
        
        if max_records is not None and len(full_dates) > max_records:
            full_dates = full_dates[-max_records:]

        # 2. Expansion
        calendar_dict = current_group_info.copy()
        calendar_dict[date_column] = full_dates
        full_calendar = pd.DataFrame(calendar_dict)

        # 3. Merge
        merged = full_calendar.merge(group, on=group_columns + [date_column], how="left")
        
        total_len = len(merged)
        interpolated_count = merged[variable].isna().sum()
        interpolation_rate = interpolated_count / total_len if total_len > 0 else 0

        # --- Check 1: Min Records ---
        if raw_record_count < min_records:
            drop_entry = current_group_info.copy()
            drop_entry.update({
                "reason": "Below Min Records",
                "details": f"Total records {raw_record_count} < {min_records}",
                "record_count": raw_record_count,  # Added raw record count
                #"interpolated_records": total_len       # This is the 'interpolated' length
            })
            dropped_metrics.append(drop_entry)
            continue

        # --- Check 2: Max Interpolation Rate ---
        if interpolation_rate > 0.25:
            drop_entry = current_group_info.copy()
            drop_entry.update({
                "reason": "High Interpolation",
                "details": f"{interpolation_rate:.1%} > 25%",
                "record_count": raw_record_count,  # Added raw record count
                #"interpolated_records": total_len       # This is the 'interpolated' length
            })
            dropped_metrics.append(drop_entry)
            continue

        # --- Success: Interpolate ---
        merged["is_missing_record"] = merged[variable].isna()
        merged[variable] = merged[variable].interpolate(method="linear", limit_direction="both")

        success_entry = current_group_info.copy()
        success_entry.update({
            "initial_records": raw_record_count,
            "interpolated_records": interpolated_count,
            "final_records": total_len,
            "interpolation_pct": round(interpolation_rate * 100, 2)
        })
        success_metrics.append(success_entry)
        full_group_data.append(merged)

    # Convert lists of dicts to DataFrames
    final_df = pd.concat(full_group_data, ignore_index=True) if full_group_data else pd.DataFrame()
    success_report = pd.DataFrame(success_metrics)
    exclusion_report = pd.DataFrame(dropped_metrics)
    
    return final_df, success_report, exclusion_report


def print_anomaly_stats(df, final_results, success_report, exclusion_report, group_columns, interpolation_method="linear"):
    # --- 1. DATA PREPARATION & CALCULATIONS ---
    total_records = len(df)
    Evaluated_records = len(final_results)
    total_anomalies = final_results['is_Anomaly'].fillna(False).astype(bool).sum()
    anomaly_rate = (total_anomalies / total_records) * 100 if total_records > 0 else 0

    num_excluded = len(exclusion_report)
    total_groups = len(success_report) + num_excluded
    evaluated_groups = len(success_report)
    
    # Interpolation stats
    total_interpolated_records = success_report['interpolated_records'].sum() if not success_report.empty else 0
    groups_with_interpolation = success_report[success_report['interpolated_records'] > 0].shape[0] if not success_report.empty else 0

    # Exclusion stats
    if num_excluded > 0:
        high_interp_mask = exclusion_report['reason'] == "High Interpolation"
        min_history_mask = exclusion_report['reason'] == "Below Min Records"
        
        groups_not_interp = high_interp_mask.sum()
        records_not_interp = exclusion_report.loc[high_interp_mask, 'record_count'].sum()
        
        groups_less_min = min_history_mask.sum()
        records_less_min = exclusion_report.loc[min_history_mask, 'record_count'].sum()
    else:
        groups_not_interp = records_not_interp = groups_less_min = records_less_min = 0

    # --- 2. PRINTING UTILITIES ---
    def print_header(title):
        """Prints a standardized, high-visibility section header."""
        print("\n" + "="*55)
        print(f"{title:^55}")
        print("="*55)

    def print_stats_rows(stats):
        """Prints key-value pairs in a aligned format."""
        for label, val in stats:
            print(f"{label:<30} : {val:>22}")

    # --- 3. EXECUTION SUMMARY ---
    print_header("ANOMALY DETECTION EXECUTIVE SUMMARY")
    exec_stats = [
        ["Total Groups", f"{total_groups:,}"],
        ["Total Records", f"{total_records:,}"],
        ["Evaluated Groups", f"{evaluated_groups:,}"],
        ["Evaluated Records", f"{Evaluated_records:,}"],
        ["Detected Anomalies", f"{total_anomalies:,}"],
        ["Anomaly Rate", f"{anomaly_rate:.2f}%"]
    ]
    print_stats_rows(exec_stats)

    # --- 4. INTERPOLATION REPORT ---
    print_header("INTERPOLATION REPORT")
    interp_stats = [
        ["Interpolation Method", interpolation_method.capitalize()],
        ["Groups with Missing Records", f"{groups_with_interpolation:,}"],
        ["Total Missing Records", f"{total_interpolated_records:,}"],
        ["Successfully Interpolated", f"{total_interpolated_records:,}"],
        ["Not Interpolated Records", f"{records_not_interp:,}"]
    ]
    print_stats_rows(interp_stats)

    # --- 5. EXCLUSION SUMMARY ---
    print_header("EXCLUSION SUMMARY")
    excl_stats = [
        ["Groups Not Interpolated", f"{groups_not_interp:,}"],
        ["Records Not Interpolated", f"{records_not_interp:,}"],
        ["Groups < Min Required", f"{groups_less_min:,}"],
        ["Records < Min Required", f"{records_less_min:,}"]
    ]
    print_stats_rows(excl_stats)
    
    print(f"\nSee exclusion_report for full list of {', '.join(group_columns)}s excluded")

    # --- 6. TOP ANOMALIES & SAMPLES ---
    # Show 5 examples of interpolated records
    if 'is_interpolated' in final_results.columns:
        interpolated_samples = final_results[final_results['is_interpolated'] == True].head(5)
        if not interpolated_samples.empty:
            print("\n" + "-"*55)
            print("EXAMPLE INTERPOLATED RECORDS:")
            cols_to_show = group_columns + ['ds', 'y']
            print(interpolated_samples[cols_to_show].to_string(index=False))
            print("-" * 55)

    print(f"\nTOP 5 GROUPS BY ANOMALY RATE ({' > '.join(group_columns)}):")
    group_stats = final_results.groupby(group_columns)['is_Anomaly'].agg(['mean', 'sum']).sort_values(by='mean', ascending=False).head(5)
    
    for label, row in group_stats.iterrows():
        group_label = label if isinstance(label, (str, int)) else " | ".join(map(str, label))
        print(f" - {group_label} : {row['mean']*100:>6.2f}% ({int(row['sum'])} anomalies)")
    
    print("\n" + "="*55 + "\n")
    

def calculate_ensemble_scores(df, variable):
    """
    Calculates the normalized consensus score across all anomaly models.
    """
    
    # Identify all columns that are model flags (is_..._anomaly)
    anomaly_flags = [col for col in df.columns if col.startswith('is_') and col.endswith('_anomaly') and col != 'is_Anomaly']
    
    # 1. Total Votes (Count of True)
    df['Anomaly_Votes'] = df[anomaly_flags].sum(axis=1).astype(int)

    # 2. Total Models active for that row (Count of non-NaN values)
    df['Vote_Cnt'] = df[anomaly_flags].notna().sum(axis=1).astype(int)
    
    # 3. Anomaly Votes Score Display (x out of N)
    df['Anomaly_Votes_Display'] = df['Anomaly_Votes'].astype(int).astype(str) + " out of " + df['Vote_Cnt'].astype(int).astype(str)
    
    # 5. Final Boolean Consensus (e.g., majority rule)
    df['is_Anomaly'] = df['Anomaly_Votes'] / df['Vote_Cnt'] >= 0.5

    # 6. Scale all the model scores to be between -1 and 1
    try:
        df['Percentile_score_scaled'] = np.where(df['is_Percentile_anomaly'].isna()==False,
                                                 abs(df[variable] - (df['Percentile_high'] + df['Percentile_low'])/2)/((df['Percentile_high'] - df['Percentile_low'])/2) - 1,
                                                 np.nan)
        df['Percentile_score_scaled'] = df['Percentile_score_scaled']/abs(df['Percentile_score_scaled']).max()                                                       
    except:
        pass

    try:
        df['SD_score_scaled'] = np.where(df['is_SD_anomaly'].isna()==False,
                                         abs(df[variable] - (df['SD2_high'] + df['SD2_low'])/2)/((df['SD2_high'] - df['SD2_low'])/2) - 1,
                                         np.nan)
        df['SD_score_scaled'] = df['SD_score_scaled']/abs(df['SD_score_scaled']).max()                                                       
    except:
        pass

    try:
        df['MAD_score_scaled'] = np.where(df['is_MAD_anomaly'].isna()==False,
                                          abs(df[variable] - (df['MAD_high'] + df['MAD_low'])/2)/((df['MAD_high'] - df['MAD_low'])/2) - 1,
                                          np.nan)
        df['MAD_score_scaled'] = df['MAD_score_scaled']/abs(df['MAD_score_scaled']).max()                                                       
    except:
        pass

    try:
        df['IQR_score_scaled'] = np.where(df['is_IQR_anomaly'].isna()==False,
                                          abs(df[variable] - (df['IQR_high'] + df['IQR_low'])/2)/((df['IQR_high'] - df['IQR_low'])/2) - 1,
                                          np.nan)
        df['IQR_score_scaled'] = df['IQR_score_scaled']/abs(df['IQR_score_scaled']).max()                                                       
    except:
        pass

    try:
        df['EWMA_score_scaled'] = np.where(df['is_EWMA_anomaly'].isna()==False,
                                           abs(df[variable] - (df['EWMA_high'] + df['EWMA_low'])/2)/((df['EWMA_high'] - df['EWMA_low'])/2) - 1,
                                           np.nan)
        df['EWMA_score_scaled'] = df['EWMA_score_scaled']/abs(df['EWMA_score_scaled']).max()                                                       
    except:
        pass

    try:
        df['FB_score_scaled'] = np.where(df['is_FB_anomaly'].isna()==False,
                                         abs(df[variable] - (df['FB_high'] + df['FB_low'])/2)/((df['FB_high'] - df['FB_low'])/2) - 1,
                                         np.nan)
        df['FB_score_scaled'] = df['FB_score_scaled']/abs(df['FB_score_scaled']).max()                                                       
    except:
        pass

    try:
        df['IsoForest_score_scaled'] = np.where(df['is_IsolationForest_anomaly'].isna()==False,
                                                df['IsolationForest_score'] - df['IsolationForest_score_low'],
                                                np.nan)
        df['IsoForest_score_scaled'] = df['IsoForest_score_scaled']/abs(df['IsoForest_score_scaled']).max()
    except:
        pass

    try:
        df['dbscan_score_scaled'] = np.where(df['is_DBSCAN_anomaly'].isna()==False, df['dbscan_score_high'] - df['dbscan_score'], np.nan)
        df['dbscan_score_scaled'] = df['dbscan_score_scaled']/abs(df['dbscan_score_scaled']).max()                                                       
    except:
        pass
    
    score_scaled_cols = []
    for col in df.columns.to_list():
        if '_score_scaled' in col:
            score_scaled_cols.append(col)

    df['Anomaly_Score'] = df[score_scaled_cols].mean(axis=1)
    # Rescale all non anomalies between 0 and 0.5 and anomalies between 0.5 and 1.0
    if len(df[df['is_Anomaly'] == True]) >= 1:
        # df.loc[df['is_Anomaly'] == True, 'Anomaly_Score'] = ((df.loc[df['is_Anomaly'] == True, 'Anomaly_Score'] + 1) * 0.245) + 0.51
        
        is_anomaly_min = df[df['is_Anomaly'] == True]['Anomaly_Score'].min()
        is_anomaly_max = df[df['is_Anomaly'] == True]['Anomaly_Score'].max()
        # Scale to [0, 0.49] based on actual data range
        if is_anomaly_max == is_anomaly_min:
            df.loc[df['is_Anomaly'] == True, 'Anomaly_Score'] = df.loc[df['is_Anomaly'] == True, 'Anomaly_Score'] * 0 + 0.51
        else:
            df.loc[df['is_Anomaly'] == True, 'Anomaly_Score'] = (((df.loc[df['is_Anomaly'] == True, 'Anomaly_Score'] - is_anomaly_min) / (is_anomaly_max - is_anomaly_min)) * 0.48) + 0.52
    
    if len(df[df['is_Anomaly'] == False]) >= 1:
        not_anomaly_min = df[df['is_Anomaly'] == False]['Anomaly_Score'].min()
        not_anomaly_max = df[df['is_Anomaly'] == False]['Anomaly_Score'].max()
        # Scale to [0, 0.49] based on actual data range
        if not_anomaly_max == not_anomaly_min:
            df.loc[df['is_Anomaly'] == False, 'Anomaly_Score'] = df.loc[df['is_Anomaly'] == False, 'Anomaly_Score'] * 0 # Default to 0 if constant
        else:
            df.loc[df['is_Anomaly'] == False, 'Anomaly_Score'] = ((df.loc[df['is_Anomaly'] == False, 'Anomaly_Score'] - not_anomaly_min) / (not_anomaly_max - not_anomaly_min)) * 0.48
    
    df['Anomaly_Score_Display'] = np.where(df['is_Anomaly'] == True, np.ceil(100 * df['Anomaly_Score']), np.floor(100 * df['Anomaly_Score'])).astype(int)
    
    # 7. Reposition is_Anomaly column to the end
    df['is_Anomaly'] = df.pop('is_Anomaly')

    return df



def min_records_extraction(freq,eval_period):
    freq_upper = freq.upper()
    
    if freq_upper.startswith('W'):
        annual_count = 52
    elif freq_upper.startswith('D') or freq_upper.startswith('B'):
        annual_count = 365
    elif freq_upper.startswith('M'):
        annual_count = 12
    else:
        # Fallback to weekly if custom/unknown
        annual_count = 52

    # Logic: 1 year for min, 2 years for max
    min_records = annual_count + eval_period
    #max_records = (2 * annual_count) + eval_period
    
    return min_records