
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path

# -----------------------------
# Config (edit as needed)
# -----------------------------
INPUT_CSV = "predictions/housing_preds_default_XGBoost.csv"  # Replace with your file path
Y_TRUE_COL = "y_true"
Y_PRED_COL = "y_pred"
LOWER_Q = 0.10    # lower quantile threshold
UPPER_Q = 0.90    # upper quantile threshold
N_BINS = 10       # for decile-wise bias plot

# -----------------------------
# Load data
# -----------------------------
df = pd.read_csv(INPUT_CSV)
df["error"] = df[Y_PRED_COL] - df[Y_TRUE_COL]

# -----------------------------
# Region-wise bias computation
# -----------------------------
lower_thresh = df[Y_TRUE_COL].quantile(LOWER_Q)
upper_thresh = df[Y_TRUE_COL].quantile(UPPER_Q)

def region_mask(s, low, high):
    return s <= low, (s > low) & (s < high), s >= high

lower_mask, middle_mask, upper_mask = region_mask(df[Y_TRUE_COL], lower_thresh, upper_thresh)

def summarize_region(region_df, name):
    return {
        "Region": name,
        "# Samples": int(len(region_df)),
        "Mean True Value": float(region_df[Y_TRUE_COL].mean()),
        "Mean Error (Bias)": float(region_df["error"].mean()),
        "Bias Interpretation": "Overprediction" if region_df["error"].mean() > 0 else ("Underprediction" if region_df["error"].mean() < 0 else "No bias")
    }

bias_summary = pd.DataFrame([
    summarize_region(df[lower_mask], f"Lower {int(LOWER_Q*100)}%"),
    summarize_region(df[middle_mask], f"Middle {int(100*(UPPER_Q-LOWER_Q))}%"),
    summarize_region(df[upper_mask], "Upper 10%"),
])

# Save summary
out_dir = Path("imbreg_bias_outputs")
out_dir.mkdir(exist_ok=True)
bias_summary["Mean True Value"] = bias_summary["Mean True Value"].round(2)
bias_summary["Mean Error (Bias)"] = bias_summary["Mean Error (Bias)"].round(2)
bias_summary.to_csv(out_dir / "region_bias_summary.csv", index=False)

# -----------------------------
# Plot 1: Region-wise bias bar chart
# -----------------------------
plt.figure()
plt.bar(bias_summary["Region"], bias_summary["Mean Error (Bias)"])
plt.axhline(0, linestyle="--")
plt.title("Region-wise Mean Bias")
plt.xlabel("Region")
plt.ylabel("Mean Error (Bias)")
plt.tight_layout()
plt.savefig(out_dir / "region_bias_bar.png", dpi=150)
plt.close()

# -----------------------------
# Plot 2: Decile-wise bias line chart
# -----------------------------
df_sorted = df.sort_values(Y_TRUE_COL).reset_index(drop=True)
df_sorted["decile"] = pd.qcut(df_sorted[Y_TRUE_COL], N_BINS, labels=False, duplicates="drop")
decile_bias = df_sorted.groupby("decile")["error"].mean().reset_index()
decile_centers = df_sorted.groupby("decile")[Y_TRUE_COL].mean().reset_index().rename(columns={Y_TRUE_COL:"mean_true_by_decile"})
decile_plot = decile_bias.merge(decile_centers, on="decile")

plt.figure()
plt.plot(decile_plot["mean_true_by_decile"], decile_plot["error"], marker="o")
plt.axhline(0, linestyle="--")
plt.title("Decile-wise Mean Bias vs. Mean True Value")
# plt.xticks(decile_plot["mean_true_by_decile"].values, rotation=45)
plt.xlabel("Mean True Value (per decile)")
plt.ylabel("Mean Error (Bias)")
plt.tight_layout()
plt.savefig(out_dir / "decile_bias_line.png", dpi=150)
plt.close()

# -----------------------------
# Plot 3: Error vs. True Value scatter
# -----------------------------
plt.figure()
plt.scatter(df[Y_TRUE_COL], df["error"], s=8, alpha=0.5)
plt.axhline(0, linestyle="--")
plt.title("Error vs. True Value")
plt.xlabel("True Value (y_true)")
plt.ylabel("Error (y_pred - y_true)")
plt.tight_layout()
plt.savefig(out_dir / "error_vs_true_scatter.png", dpi=150)
plt.close()

print("Saved region-wise bias outputs to:", out_dir)


# OBSERVATIONS -:
""" The bias analysis across different regions of the target distribution reveals a clear and asymmetric pattern 
in the model’s predictions. Specifically, the XGBoost model tends to overpredict for the lowest 10% of housing 
prices with a strong positive bias (≈ +11,000), underpredict for the highest 10% with a significant negative bias 
(≈ −16,000), and performs relatively well with minimal bias in the middle 80% of the price range. This is visually 
confirmed through both the region-wise bar chart and the decile-wise bias line plot, which shows a smooth transition 
from overprediction to underprediction across increasing target values. The error vs. true value scatter plot further 
illustrates this trend, highlighting that prediction errors increase in magnitude and skew negatively as housing prices
rise. These results indicate a classic regression-to-the-mean behavior, where the model pulls extreme values toward 
the center, underestimating rare high prices and inflating low ones — a common issue in imbalanced regression tasks with skewed target distributions."""