#!/usr/bin/env python3
"""
RFX-Fuse Finance Pipeline Demo: Classification Systems
=======================================================

USE CASE 2: Finance Explainability
Traditional Approach: XGBoost + SHAP + Isolation Forest (3 tools)
RFX-Fuse Approach: 1 RFX-Fuse model

This demonstrates RFX-Fuse for explainable credit decisions:

REGULATORY CONTEXT:
  - ECOA (Equal Credit Opportunity Act) requires adverse action notices
  - Fair Lending requires documentation of decision factors
  - GDPR Article 22 requires explanation of automated decisions

RFX-Fuse PROVIDES ALL 4 EXPLANATION TYPES:
  1. Overall Variable Importance: "What factors drive loan decisions globally?" (comparable to XGBoost importance)
  2. Local Variable Importance: "Why was THIS borrower denied?" (comparable to SHAP)
  3. Overall Proximity Importance: "What clusters borrowers by risk?" (unique to RFX-Fuse)
  4. Local Proximity Importance: "Why is this borrower similar to others?" (unique to RFX-Fuse)

Comparisons included:
- RFX-Fuse vs XGBoost: Classification accuracy, feature importance
- RFX-Fuse vs SHAP: Local importance explanations
- RFX-Fuse vs Isolation Forest: Outlier detection

Dataset: Kaggle Credit Score (100K borrowers, 15 features, 29% default rate)
"""

import sys
import time
import numpy as np
from pathlib import Path

# Use non-interactive backend for saving plots without display
import matplotlib
matplotlib.use('Agg')

# Setup paths relative to this file
SCRIPT_DIR = Path(__file__).parent.resolve()
PROJECT_ROOT = SCRIPT_DIR.parent.parent
sys.path.insert(0, str(PROJECT_ROOT / 'python'))
import RFXFuse as rfx

# Optional imports for comparisons
try:
    import xgboost as xgb
    HAS_XGBOOST = True
except ImportError:
    HAS_XGBOOST = False
    print("WARNING:XGBoost not available - skipping XGBoost comparisons")

try:
    import shap
    HAS_SHAP = True
except ImportError:
    HAS_SHAP = False
    print("WARNING:SHAP not available - skipping SHAP comparisons")

try:
    import faiss
    HAS_FAISS = True
except ImportError:
    HAS_FAISS = False
    print("WARNING:FAISS not available - skipping FAISS comparisons")

# =============================================================================
# Configuration
# =============================================================================
N_TREES_RFX = 500  # RF doesn't overfit with more trees
N_TREES_XGB = 200  # XGBoost uses early stopping anyway
MAX_SAMPLES = 0  # Use all 100K samples
TOP_K = 10
USE_GPU = True
GPU_BATCH_SIZE = 50

# Override MAX_SAMPLES from command line: python demo_rfx_finance_pipeline.py 0
# Or load saved models: python demo_rfx_finance_pipeline.py 0 --load
if len(sys.argv) > 1:
    MAX_SAMPLES = int(sys.argv[1])
    print(f"[CLI] MAX_SAMPLES = {MAX_SAMPLES} (0 = all samples)")

LOAD_MODELS = '--load' in sys.argv or '-l' in sys.argv
FORCE_RETRAIN = '--retrain' in sys.argv or '-r' in sys.argv

# Model save directory (same folder as script)
MODEL_DIR = SCRIPT_DIR

# =============================================================================
# Data Loading
# =============================================================================
def load_finance_cached(max_samples=5000):
    """Load finance data from cached NPZ file."""
    cache_dir = PROJECT_ROOT / "data" / "cache"

    cache_file = cache_dir / f"claim10_finance_s{max_samples}.npz"
    if not cache_file.exists():
        cache_file = cache_dir / "claim10_finance_s0.npz"
    if not cache_file.exists():
        print(f"ERROR: No finance cache found in {cache_dir}")
        print("Please run test_arxiv_paper_claim10_finance.py first to generate cache.")
        sys.exit(1)

    print(f"Loading cached finance data from {cache_file.name}...")
    data = np.load(cache_file, allow_pickle=True)

    X = data['X'].astype(np.float32)
    y = data['y'].astype(np.int32)  # 1 = default, 0 = non-default
    feature_names = [str(f) for f in data['feature_names']]

    if max_samples > 0 and len(X) > max_samples:
        X = X[:max_samples]
        y = y[:max_samples]

    print(f"Loaded {len(X):,} borrowers, {len(feature_names)} features")
    print(f"Default rate: {y.mean()*100:.1f}%")

    return X, y, feature_names


def split_finance_data(X, y, train_frac=0.8, random_state=42):
    """Split finance data: 80% of defaulters + matched non-defaulters for training, rest for test."""
    np.random.seed(random_state)

    # Separate classes
    defaulted_idx = np.where(y == 1)[0]
    non_defaulted_idx = np.where(y == 0)[0]

    print(f"Total defaulters: {len(defaulted_idx):,}, Non-defaulters: {len(non_defaulted_idx):,}")

    # Shuffle each group
    np.random.shuffle(defaulted_idx)
    np.random.shuffle(non_defaulted_idx)

    # Take 80% of defaulters for training
    n_default_train = int(len(defaulted_idx) * train_frac)
    # Match with equal non-defaulters
    n_non_default_train = n_default_train

    # Training: balanced (80% defaulters + matched non-defaulters)
    train_idx = np.concatenate([
        defaulted_idx[:n_default_train],
        non_defaulted_idx[:n_non_default_train]
    ])

    # Test: remaining defaulters + remaining non-defaulters
    test_idx = np.concatenate([
        defaulted_idx[n_default_train:],
        non_defaulted_idx[n_non_default_train:]
    ])

    print(f"Balanced training: {n_default_train:,} defaulters + {n_non_default_train:,} non-defaulters")

    # Shuffle the combined indices
    np.random.shuffle(test_idx)
    np.random.shuffle(train_idx)

    X_train, y_train = X[train_idx], y[train_idx]
    X_test, y_test = X[test_idx], y[test_idx]

    print(f"Train set: {len(X_train):,} samples (default rate: {y_train.mean()*100:.1f}%)")
    print(f"Test set: {len(X_test):,} samples (default rate: {y_test.mean()*100:.1f}%)")

    return X_train, X_test, y_train, y_test, train_idx, test_idx


def fmt(val, width=8):
    """Format a float for display."""
    return f"{val:.4f}".rjust(width)


def fmt_sign(val, width=8):
    """Format with sign."""
    sign = "+" if val > 0 else ""
    return f"{sign}{val:.4f}".rjust(width)


# =============================================================================
# Main Demo
# =============================================================================
def main():
    print("=" * 75)
    print("RFX-FUSE FINANCE PIPELINE: Loan Denial Explanation")
    print("Regulatory Compliance: ECOA, Fair Lending, GDPR Article 22")
    print("=" * 75)

    # Load data
    X, y, feature_names = load_finance_cached(MAX_SAMPLES)

    # Split into train/test
    print("\n" + "=" * 75)
    print("TRAIN/TEST SPLIT")
    print("=" * 75)
    X_train, X_test, y_train, y_test, train_idx, test_idx = split_finance_data(X, y, train_frac=0.8)
    
    # =========================================================================
    # Find good example borrowers from TEST SET (for realistic evaluation)
    # =========================================================================
    print("\n" + "=" * 75)
    print("SELECTING EXAMPLE BORROWERS FROM TEST SET")
    print("=" * 75)

    # Cache file for example borrower indices (ensures consistency with --load)
    indices_cache_path = MODEL_DIR / f"example_indices_s{MAX_SAMPLES}.npz"

    if LOAD_MODELS and not FORCE_RETRAIN and indices_cache_path.exists():
        # Load cached indices for consistent results
        print(f"[LOAD]Loading cached example borrower indices from {indices_cache_path.name}...")
        cached = np.load(indices_cache_path)
        query_denied_idx = int(cached['query_denied_idx'])
        query_approved_idx = int(cached['query_approved_idx'])
        print(f"Loaded: denied_idx={query_denied_idx}, approved_idx={query_approved_idx}")
    else:
        # Select compelling examples from TRAINING set (where we have local importance)
        print("Selecting example borrowers from training set...")

        # Quick pre-train to get predictions on training set
        pre_clf = rfx.RandomForestClassifier(ntree=100, mtry=5, minndsize=3, use_casewise=True, use_gpu=USE_GPU, batch_size=GPU_BATCH_SIZE, iseed=42)
        pre_clf.fit(X_train, y_train)
        predictions_train = pre_clf.predict_proba(X_train)
        pred_default_proba_train = predictions_train[:, 1] if predictions_train.shape[1] > 1 else predictions_train[:, 0]

        # 1. Denied borrower: TRUE POSITIVE from TRAINING set (defaulted + model predicts high risk)
        defaulted_train_idx = np.where(y_train == 1)[0]
        high_risk_defaulters = defaulted_train_idx[pred_default_proba_train[defaulted_train_idx] > 0.6]
        if len(high_risk_defaulters) > 0:
            delay_col = feature_names.index('Num_of_Delayed_Payment') if 'Num_of_Delayed_Payment' in feature_names else 8
            reasonable_mask = X_train[high_risk_defaulters, delay_col] < 100
            if reasonable_mask.sum() > 0:
                candidates = high_risk_defaulters[reasonable_mask]
                query_denied_idx = candidates[np.argmax(pred_default_proba_train[candidates])]
            else:
                query_denied_idx = high_risk_defaulters[np.argmax(pred_default_proba_train[high_risk_defaulters])]
        else:
            query_denied_idx = defaulted_train_idx[np.argmax(pred_default_proba_train[defaulted_train_idx])]

        # 2. Approved borrower: TRUE NEGATIVE from TRAINING set
        non_defaulted_train_idx = np.where(y_train == 0)[0]
        low_risk_good = non_defaulted_train_idx[pred_default_proba_train[non_defaulted_train_idx] < 0.3]
        if len(low_risk_good) > 0:
            income_col = feature_names.index('Annual_Income') if 'Annual_Income' in feature_names else 1
            income_reasonable = X_train[low_risk_good, income_col] < 200000
            if income_reasonable.sum() > 0:
                candidates = low_risk_good[income_reasonable]
                query_approved_idx = candidates[np.argmin(pred_default_proba_train[candidates])]
            else:
                query_approved_idx = low_risk_good[np.argmin(pred_default_proba_train[low_risk_good])]
        else:
            query_approved_idx = non_defaulted_train_idx[np.argmin(pred_default_proba_train[non_defaulted_train_idx])]

        del pre_clf  # Free memory

        # Save indices for consistency with --load
        print(f"[SAVE]Saving example borrower indices to {indices_cache_path.name}...")
        np.savez(indices_cache_path,
                 query_denied_idx=query_denied_idx,
                 query_approved_idx=query_approved_idx)
        print(f"Saved: denied_idx={query_denied_idx}, approved_idx={query_approved_idx}")

    print(f"\n{'='*75}")
    print("QUERY BORROWERS (FROM TRAINING SET - for local importance access)")
    print("=" * 75)

    print(f"\n  DENIED Borrower (train_idx={query_denied_idx}) - DEFAULTED")
    print(f"     Key features:")
    for feat in ['Age', 'Annual_Income', 'Outstanding_Debt', 'Num_of_Delayed_Payment', 'Credit_Utilization_Ratio']:
        if feat in feature_names:
            idx = feature_names.index(feat)
            val = X_train[query_denied_idx, idx]
            print(f"       {feat}: {val:,.1f}")

    print(f"\n  APPROVED Borrower (train_idx={query_approved_idx}) - Non-Defaulter")
    print(f"     Key features:")
    for feat in ['Age', 'Annual_Income', 'Outstanding_Debt', 'Num_of_Delayed_Payment', 'Credit_Utilization_Ratio']:
        if feat in feature_names:
            idx = feature_names.index(feat)
            val = X_train[query_approved_idx, idx]
            print(f"       {feat}: {val:,.1f}")
    
    # =========================================================================
    # RFX-Fuse Classifier (Default Prediction)
    # =========================================================================
    print(f"\n{'='*75}")
    print("RFX-FUSE CLASSIFIER (Default Prediction / Loan Decision)")
    print("=" * 75)
    print("'Predict default risk with full explainability for regulatory compliance'")
    
    clf_model_path = MODEL_DIR / f"classifier_s{MAX_SAMPLES}.rfx"
    
    if LOAD_MODELS and not FORCE_RETRAIN and clf_model_path.exists():
        print(f"\n[LOAD]Loading saved classifier model from {clf_model_path.name}...")
        clf = rfx.load(str(clf_model_path))
        clf_time = 0.0  # No training time if loaded
        print("Model loaded successfully")
    else:
        t0 = time.time()
        clf = rfx.RandomForestClassifier(
            ntree=N_TREES_RFX,
            mtry=5,  # p/3 instead of sqrt(p) for better classification
            minndsize=3,  # Finer splits for large dataset
            use_casewise=True,  # Casewise importance
            use_gpu=USE_GPU,
            batch_size=GPU_BATCH_SIZE,
            compute_proximity_importance=True,
            compute_local_importance=True,
            compute_leaf_assignments=True,
            iseed=42,
        )
        clf.fit(X_train, y_train)  # Train on training data only (fair comparison)
        clf_time = time.time() - t0
        
        # Save model
        print(f"\n[SAVE]Saving classifier model to {clf_model_path.name}...")
        clf.save(str(clf_model_path))
        print("Model saved successfully")
    
    print(f"\nTraining: {clf_time:.1f}s ({N_TREES_RFX} trees)")
    print(f"OOB Error: {clf.get_oob_error()*100:.1f}%")
    print(f"Accuracy: {(1-clf.get_oob_error())*100:.1f}%")
    
    # Overall Variable Importance
    overall_var = clf.feature_importances_()
    sorted_var = np.argsort(overall_var)[::-1]
    
    print(f"\nOVERALL VARIABLE IMPORTANCE")
    print(f"   'What factors drive default risk globally?'")
    for i in range(5):
        idx = sorted_var[i]
        print(f"   {i+1}. {feature_names[idx]:28s} {fmt(overall_var[idx])}")
    
    # Overall Proximity Importance (prediction space)
    overall_prox_clf = clf.get_proximity_importance()
    overall_prox_clf_sum = overall_prox_clf.sum(axis=0)
    sorted_prox = np.argsort(overall_prox_clf_sum)[::-1]
    
    print(f"\nOVERALL PROXIMITY IMPORTANCE (Prediction Space)")
    print(f"   'What clusters borrowers by default risk?'")
    for i in range(5):
        idx = sorted_prox[i]
        print(f"   {i+1}. {feature_names[idx]:28s} {fmt(overall_prox_clf_sum[idx])}")
    
    # On-demand outlier detection for classifier (using leaf assignments)
    print(f"\nOUTLIER DETECTION (On-Demand, Using Leaf Assignments)")
    print(f"   Computing outlier scores on-demand for classifier...")
    t0_outlier_clf = time.time()
    outlier_scores_clf = clf.compute_outlier_scores(mode='greedy')  # Fast approximate mode
    outlier_time_clf = time.time() - t0_outlier_clf
    print(f"   Computation time: {outlier_time_clf:.2f}s (greedy mode)")
    
    # Top outliers in prediction space (classifier has no synthetic samples, but check bounds anyway)
    top_outliers_clf, top_scores_clf = clf.compute_outliers(k=5)
    print(f"\n   Top 5 Outliers in Prediction Space (score > 10 = outlier):")
    count = 0
    for idx, score in zip(top_outliers_clf, top_scores_clf):
        if idx < len(y) and idx < len(X):  # Bounds check
            outcome = "DEFAULTED" if y[idx] == 1 else "Paid"
            pred_proba = clf.predict_proba(X[idx:idx+1])[0]
            pred_risk = pred_proba[1] if len(pred_proba) > 1 else pred_proba[0]
            print(f"   {count+1}. Borrower {idx}: score={score:.1f}, P(Default)={pred_risk*100:.0f}% - {outcome}")
            count += 1
            if count >= 5:
                break
    
    # Local importance
    local_var = clf.get_local_importance()
    
    # =========================================================================
    # Example 1: DENIED Borrower (Defaulted)
    # =========================================================================
    print(f"\n{'='*75}")
    print("EXAMPLE 1: DENIED BORROWER (Defaulted)")
    print("=" * 75)
    
    pred_proba = clf.predict_proba(X_train[query_denied_idx:query_denied_idx+1])[0]
    pred_default = pred_proba[1] if len(pred_proba) > 1 else pred_proba[0]
    decision = "DENIED" if pred_default > 0.5 else "APPROVED"
    
    print(f"\n   DECISION: {decision}")
    print(f"   P(Default): {pred_default*100:.1f}%")
    print(f"   ACTUAL: {'DEFAULTED' if y_train[query_denied_idx] == 1 else 'Non-Defaulter'}")
    
    # Local Variable Importance - Adverse Action Reasons
    if local_var is not None:
        local_var_denied = local_var[query_denied_idx]
        sorted_lv = np.argsort(local_var_denied)[::-1]  # Most positive = most risky
        
        print(f"\n   ADVERSE ACTION NOTICE (ECOA Requirement)")
        print(f"   'Principal reasons for denial:'")
        count = 0
        for idx in sorted_lv:
            if local_var_denied[idx] > 0 and count < 4:  # Top 4 reasons per ECOA
                val = X_train[query_denied_idx, idx]
                print(f"   {count+1}. {feature_names[idx]:24s} = {val:>10,.1f}  (impact: {fmt_sign(local_var_denied[idx])})")
                count += 1
    
    # Top-K Similar (from classifier proximity)
    # Get more than TOP_K to ensure we have enough real samples
    similar_denied, scores_denied = clf.get_top_k_similar(query_denied_idx, TOP_K * 3)
    
    # Key features to display for similar borrowers
    key_feats = ['Age', 'Annual_Income', 'Outstanding_Debt', 'Num_of_Delayed_Payment']
    key_feat_idx = [feature_names.index(f) if f in feature_names else -1 for f in key_feats]
    
    print(f"\n   SIMILAR BORROWERS (Feature Space):")
    n_defaulted = 0
    n_shown = 0
    for sim_idx, score in zip(similar_denied, scores_denied):
        if sim_idx >= len(y_train):  # Skip synthetic samples
            continue
        if n_shown >= TOP_K:
            break
        # score is already normalized (0 to 1) from get_top_k_similar()
        proximity = score  # Already normalized proximity value
        pct = proximity * 100  # Percentage
        outcome = "DEFAULTED" if y_train[sim_idx] == 1 else "Paid"
        if y_train[sim_idx] == 1:
            n_defaulted += 1
        sim_proba = clf.predict_proba(X_train[sim_idx:sim_idx+1])[0]
        sim_risk = sim_proba[1] if len(sim_proba) > 1 else sim_proba[0]

        # Show key feature values
        feat_str = ", ".join([f"{key_feats[i][:3]}={X_train[sim_idx, key_feat_idx[i]]:,.0f}"
                              for i in range(len(key_feats)) if key_feat_idx[i] >= 0])
        print(f"   {n_shown+1}. Borrower {sim_idx} (proximity={proximity:.3f}, {pct:.1f}%) - P(Default)={sim_risk*100:.0f}% - {outcome}")
        print(f"      [{feat_str}]")
        n_shown += 1
    print(f"   → {n_defaulted}/{n_shown} similar borrowers also defaulted")
    
    # Local Proximity Importance (from classifier)
    local_prox_denied = overall_prox_clf[query_denied_idx]  # Use classifier's proximity importance
    sorted_lp = np.argsort(local_prox_denied)[::-1]
    
    print(f"\n   WHY SIMILAR (Feature Space):")
    for i in range(4):
        idx = sorted_lp[i]
        if local_prox_denied[idx] > 0:
            val = X_train[query_denied_idx, idx]
            print(f"   {i+1}. {feature_names[idx]:24s} = {val:>10,.1f}  (prox: {fmt(local_prox_denied[idx])})")
    
    # =========================================================================
    # Example 2: APPROVED Borrower (Non-Defaulter)
    # =========================================================================
    print(f"\n{'='*75}")
    print("EXAMPLE 2: APPROVED BORROWER (Non-Defaulter)")
    print("=" * 75)
    
    pred_proba = clf.predict_proba(X_train[query_approved_idx:query_approved_idx+1])[0]
    pred_default = pred_proba[1] if len(pred_proba) > 1 else pred_proba[0]
    decision = "DENIED" if pred_default > 0.5 else "APPROVED"
    
    print(f"\n   DECISION: {decision}")
    print(f"   P(Default): {pred_default*100:.1f}%")
    print(f"   ACTUAL: {'DEFAULTED' if y_train[query_approved_idx] == 1 else 'Non-Defaulter'}")
    
    # Local Variable Importance - Approval Rationale
    if local_var is not None:
        local_var_approved = local_var[query_approved_idx]
        sorted_lv = np.argsort(local_var_approved)  # Most negative = most protective
        
        print(f"\n   APPROVAL RATIONALE")
        print(f"   'Key factors supporting approval:'")
        count = 0
        for idx in sorted_lv:
            if local_var_approved[idx] < 0 and count < 4:  # Negative = reduces default risk
                val = X_train[query_approved_idx, idx]
                print(f"   {count+1}. {feature_names[idx]:24s} = {val:>10,.1f}  (impact: {fmt_sign(local_var_approved[idx])})")
                count += 1
    
    # Top-K Similar (from classifier proximity)
    similar_approved, scores_approved = clf.get_top_k_similar(query_approved_idx, TOP_K * 3)
    
    print(f"\n   SIMILAR BORROWERS (Feature Space):")
    n_paid = 0
    n_shown = 0
    for sim_idx, score in zip(similar_approved, scores_approved):
        if sim_idx >= len(y_train):  # Skip synthetic samples
            continue
        if n_shown >= TOP_K:
            break
        # score is already normalized (0 to 1) from get_top_k_similar()
        proximity = score  # Already normalized proximity value
        pct = proximity * 100  # Percentage
        outcome = "DEFAULTED" if y_train[sim_idx] == 1 else "Paid"
        if y_train[sim_idx] == 0:
            n_paid += 1
        sim_proba = clf.predict_proba(X_train[sim_idx:sim_idx+1])[0]
        sim_risk = sim_proba[1] if len(sim_proba) > 1 else sim_proba[0]

        # Show key feature values
        feat_str = ", ".join([f"{key_feats[i][:3]}={X_train[sim_idx, key_feat_idx[i]]:,.0f}"
                              for i in range(len(key_feats)) if key_feat_idx[i] >= 0])
        print(f"   {n_shown+1}. Borrower {sim_idx} (proximity={proximity:.3f}, {pct:.1f}%) - P(Default)={sim_risk*100:.0f}% - {outcome}")
        print(f"      [{feat_str}]")
        n_shown += 1
    print(f"   → {n_paid}/{n_shown} similar borrowers also paid successfully")
    
    # Local Proximity Importance (from classifier)
    local_prox_approved = overall_prox_clf[query_approved_idx]  # Use classifier's proximity importance
    sorted_lp = np.argsort(local_prox_approved)[::-1]
    
    print(f"\n   WHY SIMILAR (Feature Space):")
    for i in range(4):
        idx = sorted_lp[i]
        if local_prox_approved[idx] > 0:
            val = X_train[query_approved_idx, idx]
            print(f"   {i+1}. {feature_names[idx]:24s} = {val:>10,.1f}  (prox: {fmt(local_prox_approved[idx])})")
    
    # =========================================================================
    # =========================================================================
    # COMPARISONS: RFX vs Traditional Tools (XGBoost + SHAP + Isolation Forest)
    # =========================================================================
    print(f"\n{'='*75}")
    print("COMPARISONS: RFX-Fuse vs Traditional Tools")
    print("=" * 75)
    print("Traditional Approach: XGBoost + SHAP + Isolation Forest (3 tools)")
    print("RFX-Fuse Approach: One model object providing all capabilities")
    
    # Comparison 1: XGBoost (Classification)
    if HAS_XGBOOST:
        print(f"\nCOMPARISON 1: RFX-Fuse vs XGBoost (Classification)")
        print(f"   Training XGBoost Classifier on TRAINING SET (fair comparison)...")
        t0 = time.time()

        # Create validation split from training data (NOT test data) for early stopping
        from sklearn.model_selection import train_test_split
        X_train_xgb, X_val_xgb, y_train_xgb, y_val_xgb = train_test_split(
            X_train, y_train, test_size=0.2, random_state=42, stratify=y_train
        )

        xgb_model = xgb.XGBClassifier(
            n_estimators=N_TREES_XGB,
            max_depth=10,
            learning_rate=0.1,
            random_state=42,
            early_stopping_rounds=20,
            eval_metric='logloss'
        )
        # Train on training subset, early-stop on validation (NOT test data)
        xgb_model.fit(X_train_xgb, y_train_xgb, eval_set=[(X_val_xgb, y_val_xgb)], verbose=False)
        xgb_train_time = time.time() - t0

        # Evaluate on TEST SET (not training data!)
        xgb_pred_test = xgb_model.predict(X_test)
        xgb_accuracy_test = np.mean(xgb_pred_test == y_test)

        # RFX test accuracy (for fair comparison)
        clf_pred_test = clf.predict(X_test)
        clf_accuracy_test = np.mean(clf_pred_test == y_test)

        print(f"   {'Metric':<20s} {'RFX':>15s} {'XGBoost':>15s}")
        print(f"   {'-'*20} {'-'*15} {'-'*15}")
        print(f"   {'Train Time':<20s} {clf_time:>14.2f}s {xgb_train_time:>14.2f}s")
        print(f"   {'Test Accuracy':<20s} {clf_accuracy_test:>15.4f} {xgb_accuracy_test:>15.4f}")
        print(f"   {'OOB Error':<20s} {clf.get_oob_error():>15.4f} {'N/A (no OOB)':>15s}")
        print(f"   Note: XGBoost early stopped at {xgb_model.best_iteration+1} trees")
        
        # Feature importance comparison
        xgb_importance = xgb_model.feature_importances_
        xgb_importance_norm = xgb_importance / xgb_importance.sum()
        rfx_importance_norm = overall_var / overall_var.sum()
        
        print(f"\n   Top-5 Feature Importance Comparison:")
        print(f"   {'Rank':<6s} {'Feature':<15s} {'RFX':>12s} {'XGBoost':>12s}")
        print(f"   {'-'*6} {'-'*15} {'-'*12} {'-'*12}")
        rfx_rank = np.argsort(overall_var)[::-1]
        xgb_rank = np.argsort(xgb_importance)[::-1]
        for i in range(5):
            rfx_idx = rfx_rank[i]
            xgb_idx = xgb_rank[i]
            print(f"   {i+1:<6d} {feature_names[rfx_idx]:<15s} {rfx_importance_norm[rfx_idx]:>12.4f} {xgb_importance_norm[xgb_idx]:>12.4f}")
    
    # Comparison 2: SHAP (Local Importance)
    if HAS_SHAP and HAS_XGBOOST:
        print(f"\nCOMPARISON 2: RFX-Fuse vs SHAP (Local Importance)")
        sample_idx = query_denied_idx
        sample = X[sample_idx:sample_idx+1]
        
        rfx_local = local_var[sample_idx] if local_var is not None else None
        if rfx_local is not None:
            rfx_local_abs = np.abs(rfx_local)
            rfx_local_rank = np.argsort(rfx_local_abs)[::-1]
        
        explainer = shap.TreeExplainer(xgb_model)
        shap_values = explainer.shap_values(sample)
        shap_values_flat = shap_values[1] if isinstance(shap_values, list) else shap_values[0]
        shap_abs = np.abs(shap_values_flat)
        shap_rank = np.argsort(shap_abs)[::-1]
        
        print(f"   Sample: Denied borrower (idx={sample_idx})")
        print(f"   {'Rank':<6s} {'Feature':<15s} {'RFX-Fuse Local':>15s} {'SHAP':>15s}")
        print(f"   {'-'*6} {'-'*15} {'-'*15} {'-'*15}")
        for i in range(5):
            if rfx_local is not None:
                rfx_idx = rfx_local_rank[i]
                shap_idx = shap_rank[i]
                print(f"   {i+1:<6d} {feature_names[rfx_idx]:<15s} {rfx_local[rfx_idx]:>15.4f} {shap_values_flat[shap_idx]:>15.4f}")
        
        print(f"\n   RFX-Fuse provides local importance natively (no post-hoc computation)")
        print(f"   WARNING:SHAP requires separate computation after model training")
    
    # Comparison 3: FAISS (Similarity Search)
    if HAS_FAISS:
        print(f"\nCOMPARISON 3: RFX-Fuse vs FAISS (Similarity Search)")
        query_idx = query_denied_idx
        query = X[query_idx:query_idx+1]

        t0 = time.time()
        rfx_similar, rfx_scores = clf.get_top_k_similar(query_idx, k=TOP_K)
        rfx_sim_time = time.time() - t0
        
        t0 = time.time()
        X_norm = X / (np.linalg.norm(X, axis=1, keepdims=True) + 1e-8)
        query_norm = query / (np.linalg.norm(query, axis=1, keepdims=True) + 1e-8)
        
        dimension = X.shape[1]
        index = faiss.IndexFlatIP(dimension)
        index.add(X_norm.astype('float32'))
        
        query_faiss = query_norm.astype('float32')
        distances, faiss_indices = index.search(query_faiss, TOP_K)
        faiss_sim_time = time.time() - t0
        
        print(f"   Query: Denied borrower (idx={query_idx})")
        print(f"   {'Metric':<20s} {'RFX':>15s} {'FAISS':>15s}")
        print(f"   {'-'*20} {'-'*15} {'-'*15}")
        print(f"   {'Query Time':<20s} {rfx_sim_time:>14.2f}s {faiss_sim_time:>14.2f}s")
        print(f"   {'Explanations':<20s} {'Yes (proximity imp)':>15s} {'No':>15s}")
        print(f"   {'Top-K Results':<20s} {len(rfx_similar):>15d} {len(faiss_indices[0]):>15d}")
    
    # =========================================================================
    # CREATE COMPREHENSIVE 18-PANEL VISUALIZATION
    # =========================================================================
    print(f"\n{'='*75}")
    print("GENERATING COMPREHENSIVE 18-PANEL VISUALIZATION...")
    print("=" * 75)

    import matplotlib.pyplot as plt
    from matplotlib.gridspec import GridSpec
    from sklearn.metrics import confusion_matrix, precision_recall_curve, f1_score, precision_score, recall_score, roc_curve, roc_auc_score
    from sklearn.ensemble import IsolationForest

    # =========================================================================
    # THRESHOLD TUNING FOR RECALL (Critical for catching loan defaults)
    # IMPORTANT: All evaluation is on TEST SET to avoid overfitting appearance
    # =========================================================================
    print("\n  Tuning threshold for optimal recall (catching defaults is critical)...")
    print("  NOTE: All metrics computed on TEST SET (not training data)")

    # Get probabilities for TEST SET only (proper evaluation)
    rfx_proba_test = clf.predict_proba(X_test)
    rfx_proba_default_test = rfx_proba_test[:, 1] if rfx_proba_test.shape[1] > 1 else rfx_proba_test[:, 0]

    if HAS_XGBOOST:
        xgb_proba_test = xgb_model.predict_proba(X_test)
        xgb_proba_default_test = xgb_proba_test[:, 1] if xgb_proba_test.shape[1] > 1 else xgb_proba_test[:, 0]

    # Find optimal thresholds for different objectives (on test set)
    thresholds = np.arange(0.1, 0.9, 0.05)

    rfx_metrics = {'precision': [], 'recall': [], 'f1': []}
    xgb_metrics = {'precision': [], 'recall': [], 'f1': []} if HAS_XGBOOST else None

    for thresh in thresholds:
        rfx_pred_thresh = (rfx_proba_default_test >= thresh).astype(int)
        rfx_metrics['precision'].append(precision_score(y_test, rfx_pred_thresh, zero_division=0))
        rfx_metrics['recall'].append(recall_score(y_test, rfx_pred_thresh, zero_division=0))
        rfx_metrics['f1'].append(f1_score(y_test, rfx_pred_thresh, zero_division=0))

        if HAS_XGBOOST:
            xgb_pred_thresh = (xgb_proba_default_test >= thresh).astype(int)
            xgb_metrics['precision'].append(precision_score(y_test, xgb_pred_thresh, zero_division=0))
            xgb_metrics['recall'].append(recall_score(y_test, xgb_pred_thresh, zero_division=0))
            xgb_metrics['f1'].append(f1_score(y_test, xgb_pred_thresh, zero_division=0))

    # For loan defaults: optimize for F1 (balanced precision-recall)
    rfx_recall_optimized_thresh = thresholds[np.argmax(rfx_metrics['recall'])]
    rfx_f1_optimized_thresh = thresholds[np.argmax(rfx_metrics['f1'])]

    # Use F1-optimized threshold for each model (fair comparison)
    optimal_thresh_rfx = rfx_f1_optimized_thresh

    if HAS_XGBOOST:
        xgb_recall_optimized_thresh = thresholds[np.argmax(xgb_metrics['recall'])]
        xgb_f1_optimized_thresh = thresholds[np.argmax(xgb_metrics['f1'])]
        optimal_thresh_xgb = xgb_f1_optimized_thresh

    print(f"  RFX F1-optimized threshold: {rfx_f1_optimized_thresh:.2f}")
    if HAS_XGBOOST:
        print(f"  XGBoost F1-optimized threshold: {xgb_f1_optimized_thresh:.2f}")
    print(f"  Using each model's optimal F1 threshold for fair comparison")

    # Get predictions with optimized threshold (on TEST SET)
    rfx_pred_optimized = (rfx_proba_default_test >= optimal_thresh_rfx).astype(int)
    if HAS_XGBOOST:
        xgb_pred_optimized = (xgb_proba_default_test >= optimal_thresh_xgb).astype(int)

    # Compute metrics at optimized threshold (on TEST SET)
    rfx_precision_opt = precision_score(y_test, rfx_pred_optimized)
    rfx_recall_opt = recall_score(y_test, rfx_pred_optimized)
    rfx_f1_opt = f1_score(y_test, rfx_pred_optimized)

    if HAS_XGBOOST:
        xgb_precision_opt = precision_score(y_test, xgb_pred_optimized)
        xgb_recall_opt = recall_score(y_test, xgb_pred_optimized)
        xgb_f1_opt = f1_score(y_test, xgb_pred_optimized)

    print(f"\n  Metrics at optimal thresholds (TEST SET):")
    print(f"    RFX      (thresh={optimal_thresh_rfx:.2f}): Precision={rfx_precision_opt:.3f}, Recall={rfx_recall_opt:.3f}, F1={rfx_f1_opt:.3f}")
    if HAS_XGBOOST:
        print(f"    XGBoost  (thresh={optimal_thresh_xgb:.2f}): Precision={xgb_precision_opt:.3f}, Recall={xgb_recall_opt:.3f}, F1={xgb_f1_opt:.3f}")

    # =========================================================================
    # CREATE 18-PANEL FIGURE (6 rows × 3 columns)
    # =========================================================================
    # FIGURE A: Predictions & Feature Importance (9 panels)
    # =========================================================================
    try:
        fig_a = plt.figure(figsize=(18, 16))
        fig_a.suptitle('RFX-Fuse Finance Pipeline (Part A): Predictions & Feature Importance\nRFX (1 Model) vs Traditional Tools (XGBoost + SHAP + Isolation Forest)',
                     fontsize=14, fontweight='bold', y=0.98)

        gs = GridSpec(3, 3, figure=fig_a, hspace=0.4, wspace=0.35)

        # =====================================================================
        # ROW 0: Dataset Description + Confusion Matrices
        # =====================================================================

        # Panel 0,0: Dataset Description
        ax_desc = fig_a.add_subplot(gs[0, 0])
        ax_desc.axis('off')

        default_rate = y.mean() * 100
        n_features = len(feature_names)

        desc_text = f"""DATASET & METHODOLOGY
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Kaggle Credit Score | {len(X):,} borrowers
Features: {n_features} | Default rate: {default_rate:.1f}%

STRATIFIED TRAIN/TEST SPLIT
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Train: {len(X_train):,} ({y_train.mean()*100:.1f}% default)
Test:  {len(X_test):,} ({y_test.mean()*100:.1f}% default)

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 RFX (1 model)     vs.  TRADITIONAL (3)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
• {N_TREES_RFX} trees, GPU       • XGBoost
• Predictions          • SHAP
• Importance           • Isolation Forest
• Outliers
• Similarity
(all from 1 model!)

THRESHOLD: RFX={optimal_thresh_rfx:.2f} (F1-optimized)
REGULATORY: ECOA + Fair Lending + GDPR"""
        if HAS_XGBOOST:
            desc_text = desc_text.replace(f"THRESHOLD: RFX={optimal_thresh_rfx:.2f}",
                                          f"THRESHOLDS: RFX={optimal_thresh_rfx:.2f}, XGB={optimal_thresh_xgb:.2f}")

        ax_desc.text(0.05, 0.95, desc_text, transform=ax_desc.transAxes,
                    fontsize=9, verticalalignment='top', fontfamily='monospace',
                    bbox=dict(boxstyle='round', facecolor='lightyellow', alpha=0.8))
        ax_desc.set_title('Dataset & Methodology', fontsize=12, fontweight='bold')

        # Panel 0,1: RFX Confusion Matrix (Recall-Optimized) - TEST SET
        ax_cm_rfx = fig_a.add_subplot(gs[0, 1])
        cm_rfx = confusion_matrix(y_test, rfx_pred_optimized)
        im_rfx = ax_cm_rfx.imshow(cm_rfx, cmap='Blues', aspect='auto')
        ax_cm_rfx.set_xticks([0, 1])
        ax_cm_rfx.set_yticks([0, 1])
        ax_cm_rfx.set_xticklabels(['Paid', 'Default'], fontsize=10)
        ax_cm_rfx.set_yticklabels(['Paid', 'Default'], fontsize=10)
        ax_cm_rfx.set_xlabel('Predicted', fontsize=11, fontweight='bold')
        ax_cm_rfx.set_ylabel('Actual', fontsize=11, fontweight='bold')

        # Add text annotations
        for i in range(2):
            for j in range(2):
                text_color = 'white' if cm_rfx[i, j] > cm_rfx.max()/2 else 'black'
                ax_cm_rfx.text(j, i, f'{cm_rfx[i, j]:,}', ha='center', va='center',
                              fontsize=14, fontweight='bold', color=text_color)

        ax_cm_rfx.set_title(f'RFX-Fuse Confusion Matrix (TEST)\nRecall={rfx_recall_opt:.3f}, Precision={rfx_precision_opt:.3f}',
                           fontsize=12, fontweight='bold')

        # Panel 0,2: XGBoost Confusion Matrix - TEST SET
        ax_cm_xgb = fig_a.add_subplot(gs[0, 2])
        if HAS_XGBOOST:
            cm_xgb = confusion_matrix(y_test, xgb_pred_optimized)
            im_xgb = ax_cm_xgb.imshow(cm_xgb, cmap='Oranges', aspect='auto')
            ax_cm_xgb.set_xticks([0, 1])
            ax_cm_xgb.set_yticks([0, 1])
            ax_cm_xgb.set_xticklabels(['Paid', 'Default'], fontsize=10)
            ax_cm_xgb.set_yticklabels(['Paid', 'Default'], fontsize=10)
            ax_cm_xgb.set_xlabel('Predicted', fontsize=11, fontweight='bold')
            ax_cm_xgb.set_ylabel('Actual', fontsize=11, fontweight='bold')

            for i in range(2):
                for j in range(2):
                    text_color = 'white' if cm_xgb[i, j] > cm_xgb.max()/2 else 'black'
                    ax_cm_xgb.text(j, i, f'{cm_xgb[i, j]:,}', ha='center', va='center',
                                  fontsize=14, fontweight='bold', color=text_color)

            ax_cm_xgb.set_title(f'XGBoost Confusion Matrix (TEST)\nRecall={xgb_recall_opt:.3f}, Precision={xgb_precision_opt:.3f}',
                               fontsize=12, fontweight='bold')
        else:
            ax_cm_xgb.text(0.5, 0.5, 'XGBoost not available', ha='center', va='center',
                          fontsize=12, transform=ax_cm_xgb.transAxes)
            ax_cm_xgb.set_title('XGBoost Confusion Matrix', fontsize=12, fontweight='bold')

        # =====================================================================
        # ROW 1: ROC/AUC + Precision-Recall + Metrics Comparison
        # =====================================================================

        # Panel 1,0: ROC/AUC Curve - TEST SET
        ax_roc = fig_a.add_subplot(gs[1, 0])

        rfx_fpr, rfx_tpr, _ = roc_curve(y_test, rfx_proba_default_test)
        rfx_auc = roc_auc_score(y_test, rfx_proba_default_test)
        ax_roc.plot(rfx_fpr, rfx_tpr, 'b-', linewidth=2, label=f'RFX-Fuse(AUC={rfx_auc:.3f})')

        if HAS_XGBOOST:
            xgb_fpr, xgb_tpr, _ = roc_curve(y_test, xgb_proba_default_test)
            xgb_auc = roc_auc_score(y_test, xgb_proba_default_test)
            ax_roc.plot(xgb_fpr, xgb_tpr, 'r--', linewidth=2, label=f'XGBoost (AUC={xgb_auc:.3f})')

        ax_roc.plot([0, 1], [0, 1], 'k--', linewidth=1, alpha=0.5, label='Random')
        ax_roc.set_xlabel('False Positive Rate', fontsize=11, fontweight='bold')
        ax_roc.set_ylabel('True Positive Rate', fontsize=11, fontweight='bold')
        ax_roc.set_title('ROC Curve: RFX vs XGBoost (TEST)\n(Area Under Curve)', fontsize=12, fontweight='bold')
        ax_roc.legend(fontsize=10, loc='lower right')
        ax_roc.grid(alpha=0.3)
        ax_roc.set_xlim(-0.02, 1.02)
        ax_roc.set_ylim(-0.02, 1.02)

        # Panel 1,1: Precision-Recall Curve - TEST SET
        ax_pr = fig_a.add_subplot(gs[1, 1])

        rfx_precision_curve, rfx_recall_curve, _ = precision_recall_curve(y_test, rfx_proba_default_test)
        ax_pr.plot(rfx_recall_curve, rfx_precision_curve, 'b-', linewidth=2, label='RFX')

        if HAS_XGBOOST:
            xgb_precision_curve, xgb_recall_curve, _ = precision_recall_curve(y_test, xgb_proba_default_test)
            ax_pr.plot(xgb_recall_curve, xgb_precision_curve, 'r--', linewidth=2, label='XGBoost')

        # Mark operating point
        ax_pr.scatter([rfx_recall_opt], [rfx_precision_opt], s=200, c='blue', marker='*',
                     zorder=5, label=f'RFX-Fuse@ {optimal_thresh_rfx:.2f}')
        if HAS_XGBOOST:
            ax_pr.scatter([xgb_recall_opt], [xgb_precision_opt], s=150, c='red', marker='^',
                         zorder=5, label=f'XGB @ {optimal_thresh_xgb:.2f}')

        ax_pr.set_xlabel('Recall (Catching Defaults)', fontsize=11, fontweight='bold')
        ax_pr.set_ylabel('Precision', fontsize=11, fontweight='bold')
        ax_pr.set_title('Precision-Recall Curve (TEST)\n(Higher recall = catch more defaults)', fontsize=12, fontweight='bold')
        ax_pr.legend(fontsize=9, loc='lower left')
        ax_pr.grid(alpha=0.3)
        ax_pr.set_xlim(0, 1.05)
        ax_pr.set_ylim(0, 1.05)

        # Panel 1,2: Precision/Recall/F1 Bar Comparison
        ax_bars = fig_a.add_subplot(gs[1, 2])
        metrics_labels = ['Precision', 'Recall', 'F1 Score']
        rfx_vals = [rfx_precision_opt, rfx_recall_opt, rfx_f1_opt]

        x = np.arange(len(metrics_labels))
        width = 0.35

        bars1 = ax_bars.bar(x - width/2, rfx_vals, width, label='RFX', color='steelblue', alpha=0.8, edgecolor='navy')

        if HAS_XGBOOST:
            xgb_vals = [xgb_precision_opt, xgb_recall_opt, xgb_f1_opt]
            bars2 = ax_bars.bar(x + width/2, xgb_vals, width, label='XGBoost', color='coral', alpha=0.8, edgecolor='darkred')

        ax_bars.set_ylabel('Score', fontsize=11, fontweight='bold')
        if HAS_XGBOOST:
            ax_bars.set_title(f'Classification Metrics @ Optimal F1 Thresholds\n(RFX={optimal_thresh_rfx:.2f}, XGB={optimal_thresh_xgb:.2f})',
                             fontsize=12, fontweight='bold')
        else:
            ax_bars.set_title(f'Classification Metrics @ Optimal F1 Threshold\n(RFX={optimal_thresh_rfx:.2f})',
                             fontsize=12, fontweight='bold')
        ax_bars.set_xticks(x)
        ax_bars.set_xticklabels(metrics_labels, fontsize=10, fontweight='bold')
        ax_bars.legend(fontsize=10)
        ax_bars.grid(axis='y', alpha=0.3)
        ax_bars.set_ylim(0, 1.1)

        # Add value labels
        for bar in bars1:
            height = bar.get_height()
            ax_bars.text(bar.get_x() + bar.get_width()/2., height + 0.02,
                        f'{height:.3f}', ha='center', va='bottom', fontsize=9, fontweight='bold')
        if HAS_XGBOOST:
            for bar in bars2:
                height = bar.get_height()
                ax_bars.text(bar.get_x() + bar.get_width()/2., height + 0.02,
                            f'{height:.3f}', ha='center', va='bottom', fontsize=9, fontweight='bold')

        # =====================================================================
        # ROW 2: Feature Importance Comparisons + Summary
        # =====================================================================

        # Panel 2,0: Overall Variable Importance (RFX vs XGBoost)
        ax_var_imp = fig_a.add_subplot(gs[2, 0])

        top_n = min(10, len(feature_names))
        rfx_rank = np.argsort(overall_var)[::-1][:top_n]
        rfx_importance_norm = overall_var / overall_var.sum()

        y_pos = np.arange(top_n)
        width = 0.35

        ax_var_imp.barh(y_pos - width/2, rfx_importance_norm[rfx_rank], width,
                       label='RFX', color='steelblue', alpha=0.8, edgecolor='navy')

        if HAS_XGBOOST:
            xgb_rank = np.argsort(xgb_importance)[::-1][:top_n]
            xgb_importance_norm = xgb_importance / xgb_importance.sum()
            ax_var_imp.barh(y_pos + width/2, xgb_importance_norm[xgb_rank], width,
                           label='XGBoost', color='coral', alpha=0.8, edgecolor='darkred')

        ax_var_imp.set_xlabel('Normalized Importance', fontsize=11, fontweight='bold')
        ax_var_imp.set_title('Overall Feature Importance\nRFX vs XGBoost (Top 10)', fontsize=12, fontweight='bold')
        ax_var_imp.set_yticks(y_pos)
        ax_var_imp.set_yticklabels([feature_names[i][:18] for i in rfx_rank], fontsize=9)
        ax_var_imp.legend(fontsize=9)
        ax_var_imp.grid(axis='x', alpha=0.3)
        ax_var_imp.invert_yaxis()

        # Panel 2,1: RFX-ONLY Overall Proximity Importance (normalized)
        ax_prox_overall = fig_a.add_subplot(gs[2, 1])

        top_n = min(10, len(feature_names))
        prox_rank = np.argsort(overall_prox_clf_sum)[::-1][:top_n]
        overall_prox_norm = overall_prox_clf_sum / (overall_prox_clf_sum.max() + 1e-10)

        y_pos = np.arange(top_n)
        ax_prox_overall.barh(y_pos, overall_prox_norm[prox_rank], color='mediumseagreen',
                            alpha=0.8, edgecolor='darkgreen', linewidth=1.5)

        ax_prox_overall.set_xlabel('Normalized Proximity Importance', fontsize=10, fontweight='bold')
        ax_prox_overall.set_title('RFX-Fuse: Overall Proximity Importance\n("What clusters borrowers?" - Not in XGBoost)',
                                 fontsize=11, fontweight='bold', color='darkgreen')
        ax_prox_overall.set_yticks(y_pos)
        ax_prox_overall.set_yticklabels([feature_names[i][:18] for i in prox_rank], fontsize=9)
        ax_prox_overall.grid(axis='x', alpha=0.3)
        ax_prox_overall.invert_yaxis()
        ax_prox_overall.set_xlim(0, 1.05)

        # Panel 2,2: Summary pointing to Figure B
        ax_summary = fig_a.add_subplot(gs[2, 2])
        ax_summary.axis('off')

        summary_text = """FIGURE B: EXPLAINABILITY
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

DEFAULTER ANALYSIS (Row 1)
• Profile of high-risk borrower
• Local importance: why predicted risky
• Top-K similar: neighbors also defaulted

NON-DEFAULTER ANALYSIS (Row 2)
• Profile of low-risk borrower
• Local importance: why predicted safe
• Top-K similar: neighbors also paid

OUTLIER DETECTION (Row 3)
• RFX-Fuse vs Isolation Forest
• Complementary methods compared

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
KEY INSIGHT: Same model explains
both outcomes intuitively!"""

        ax_summary.text(0.05, 0.95, summary_text, transform=ax_summary.transAxes,
                       fontsize=9, verticalalignment='top', fontfamily='monospace',
                       bbox=dict(boxstyle='round', facecolor='lavender', alpha=0.8))
        ax_summary.set_title('Explainability Overview', fontsize=12, fontweight='bold')

        # Save Figure A
        fig_a_path = SCRIPT_DIR / "loan_classification_9panel_a.png"
        fig_a_path.parent.mkdir(parents=True, exist_ok=True)
        plt.savefig(fig_a_path, dpi=300, bbox_inches='tight')
        print(f"\nSaved Figure A (Predictions & Importance) to: {fig_a_path}")
        plt.close(fig_a)

    except Exception as e:
        print(f"\n  Could not create Figure A: {e}")
        import traceback
        traceback.print_exc()

    # =========================================================================
    # FIGURE B: Explainability Stories (9 panels)
    # =========================================================================
    try:
        fig_b = plt.figure(figsize=(18, 18))

        # Prepare profile data for text block
        query_idx_def = query_denied_idx
        query_features_def = X_train[query_idx_def]
        pred_proba_def = clf.predict_proba(X_train[query_idx_def:query_idx_def+1])[0]
        pred_risk_def = pred_proba_def[1] if len(pred_proba_def) > 1 else pred_proba_def[0]
        actual_outcome_def = 'DEFAULTED' if y_train[query_idx_def] == 1 else 'Paid'

        query_idx_nondef = query_approved_idx
        query_features_nondef = X_train[query_idx_nondef]
        pred_proba_nondef = clf.predict_proba(X_train[query_idx_nondef:query_idx_nondef+1])[0]
        pred_risk_nondef = pred_proba_nondef[1] if len(pred_proba_nondef) > 1 else pred_proba_nondef[0]
        actual_outcome_nondef = 'DEFAULTED' if y_train[query_idx_nondef] == 1 else 'Paid'

        # Compute outlier info early
        # RFX: higher = more outlier, so use argmax
        top_outlier_idx = np.argmax(outlier_scores_clf)
        top_outlier_score = outlier_scores_clf[top_outlier_idx]
        top_outlier_actual = y_train[top_outlier_idx]
        top_outlier_outcome = 'DEFAULTED' if top_outlier_actual == 1 else 'Paid'
        pred_proba_outlier = clf.predict_proba(X_train[top_outlier_idx:top_outlier_idx+1])[0]
        pred_risk_outlier = pred_proba_outlier[1] if len(pred_proba_outlier) > 1 else pred_proba_outlier[0]

        # Main title only
        fig_b.suptitle('RFX-Fuse Finance Pipeline (Part B): Explainability Stories',
                     fontsize=14, fontweight='bold', y=0.98)

        gs = GridSpec(3, 3, figure=fig_b, hspace=0.35, wspace=0.35)

        # Add row legend as text box in upper left (centered)
        legend_text = (
            f"Row 1: DEFAULTER #{query_idx_def} (Risk={pred_risk_def*100:.0f}%, {actual_outcome_def})\n"
            f"Row 2: NON-DEFAULTER #{query_idx_nondef} (Risk={pred_risk_nondef*100:.0f}%, {actual_outcome_nondef})\n"
            f"Row 3: TOP OUTLIER #{top_outlier_idx} (Risk={pred_risk_outlier*100:.0f}%, {top_outlier_outcome})"
        )
        fig_b.text(0.01, 0.95, legend_text, transform=fig_b.transFigure,
                  fontsize=9, fontweight='bold', va='top', ha='left',
                  bbox=dict(boxstyle='round', facecolor='lightyellow', edgecolor='gray', alpha=0.9))

        # =====================================================================
        # ROW 0: DEFAULTER ANALYSIS - RFX vs SHAP + Top-K + Local Prox Imp
        # =====================================================================

        # Panel 0,0: Defaulter Local Variable Importance (RFX vs SHAP with overlap)
        ax_def_shap = fig_b.add_subplot(gs[0, 0])

        sample_def = X_train[query_idx_def:query_idx_def+1]
        rfx_top6_def = set()
        shap_top6_def = set()

        if HAS_SHAP and HAS_XGBOOST:
            try:
                rfx_local_def = local_var[query_idx_def] if local_var is not None else None

                if rfx_local_def is not None:
                    rfx_local_abs_def = np.abs(rfx_local_def)
                    rfx_local_norm_def = rfx_local_abs_def / (rfx_local_abs_def.sum() + 1e-10)
                    rfx_top6_def = set(np.argsort(rfx_local_abs_def)[-6:])

                    explainer = shap.TreeExplainer(xgb_model)
                    shap_values_def = explainer.shap_values(sample_def)
                    shap_values_flat_def = shap_values_def[1] if isinstance(shap_values_def, list) else shap_values_def[0]
                    shap_abs_def = np.abs(shap_values_flat_def).flatten()
                    shap_norm_def = shap_abs_def / (shap_abs_def.sum() + 1e-10)
                    shap_top6_def = set(np.argsort(shap_abs_def)[-6:])

                    # Calculate overlap
                    overlap_def = len(rfx_top6_def & shap_top6_def)

                    combined_importance_def = rfx_local_norm_def + shap_norm_def
                    top_feat_indices_def = np.argsort(combined_importance_def)[-6:][::-1]

                    y_pos = np.arange(len(top_feat_indices_def))
                    width = 0.35

                    ax_def_shap.barh(y_pos - width/2, rfx_local_norm_def[top_feat_indices_def], width,
                                    label='RFX-Fuse', color='steelblue', alpha=0.8, edgecolor='navy')
                    ax_def_shap.barh(y_pos + width/2, shap_norm_def[top_feat_indices_def], width,
                                    label='SHAP', color='orange', alpha=0.8, edgecolor='darkorange')

                    ax_def_shap.set_xlabel('Normalized Importance', fontsize=10, fontweight='bold')
                    ax_def_shap.set_title(f'Why Predicted Risky?\n(RFX-Fuse vs SHAP, Top-6 Agreement: {overlap_def}/6)', fontsize=10, fontweight='bold')
                    ax_def_shap.set_yticks(y_pos)
                    ax_def_shap.set_yticklabels([feature_names[i][:15] for i in top_feat_indices_def], fontsize=9)
                    ax_def_shap.legend(fontsize=8)
                    ax_def_shap.grid(axis='x', alpha=0.3)
                    ax_def_shap.invert_yaxis()
            except Exception as e:
                ax_def_shap.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center', fontsize=10)
                ax_def_shap.set_title('Local Importance', fontsize=12, fontweight='bold')
        else:
            ax_def_shap.text(0.5, 0.5, 'SHAP not available', ha='center', va='center', fontsize=11)
            ax_def_shap.set_title('Local Importance', fontsize=12, fontweight='bold')

        # Panel 0,1: Defaulter Top-K Similar
        ax_def_topk = fig_b.add_subplot(gs[0, 1])

        try:
            similar_indices_def, similar_scores_def = clf.get_top_k_similar(query_idx_def, k=TOP_K)

            similar_outcomes_def = []
            similar_risks_def = []
            for idx, score in zip(similar_indices_def, similar_scores_def):
                if idx < len(y_train):
                    risk = clf.predict_proba(X_train[idx:idx+1])[0]
                    risk_pct = (risk[1] if len(risk) > 1 else risk[0]) * 100
                    similar_outcomes_def.append(1 if y_train[idx] == 1 else 0)
                    similar_risks_def.append(risk_pct)

            if len(similar_risks_def) > 0:
                x_pos = np.arange(len(similar_risks_def))
                colors_def = ['salmon' if o == 1 else 'lightgreen' for o in similar_outcomes_def]
                ax_def_topk.bar(x_pos, similar_risks_def, color=colors_def, alpha=0.8, edgecolor='black', linewidth=1)

                ax_def_topk.axhline(50, color='red', linestyle='--', linewidth=2, label='50% threshold')
                ax_def_topk.set_xlabel('Similar Borrowers', fontsize=10, fontweight='bold')
                ax_def_topk.set_ylabel('Default Risk (%)', fontsize=10, fontweight='bold')

                n_defaulted_def = sum(similar_outcomes_def)
                ax_def_topk.set_title(f'Neighbors of Defaulter\n{n_defaulted_def}/{len(similar_outcomes_def)} also defaulted',
                                     fontsize=10, fontweight='bold')
                ax_def_topk.set_xticks(x_pos)
                ax_def_topk.set_xticklabels([f'#{i+1}' for i in range(len(similar_risks_def))], fontsize=8)
                ax_def_topk.legend(fontsize=8)
                ax_def_topk.grid(axis='y', alpha=0.3)
                ax_def_topk.set_ylim(0, 105)
        except Exception as e:
            ax_def_topk.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center', fontsize=10)
            ax_def_topk.set_title('Top-K Similar', fontsize=12, fontweight='bold')

        # Panel 0,2: Defaulter Local Proximity Importance (WHY similar to neighbors)
        ax_def_prox = fig_b.add_subplot(gs[0, 2])

        try:
            local_prox_def = overall_prox_clf[query_idx_def]
            local_prox_sorted_def = np.argsort(local_prox_def)[::-1][:8]
            local_prox_norm_def = local_prox_def / (local_prox_def.max() + 1e-10)

            y_pos = np.arange(len(local_prox_sorted_def))
            prox_vals_def = local_prox_norm_def[local_prox_sorted_def]

            bars = ax_def_prox.barh(y_pos, prox_vals_def, color='coral', alpha=0.8,
                                   edgecolor='darkred', linewidth=1.5)

            ax_def_prox.set_xlabel('Normalized Prox Importance', fontsize=10, fontweight='bold')
            ax_def_prox.set_title('Why Similar to Neighbors?\n(Local Prox Importance - RFX only)', fontsize=10, fontweight='bold')
            ax_def_prox.set_yticks(y_pos)
            ax_def_prox.set_yticklabels([feature_names[i][:15] for i in local_prox_sorted_def], fontsize=9)
            ax_def_prox.grid(axis='x', alpha=0.3)
            ax_def_prox.invert_yaxis()
            ax_def_prox.set_xlim(0, 1.05)

            # Add actual values inside bars (right-aligned, black text)
            for bar, feat_idx in zip(bars, local_prox_sorted_def):
                feat_val = X_train[query_idx_def, feat_idx]
                if feat_val == int(feat_val):
                    label = f'Actual: {int(feat_val)}'
                else:
                    label = f'Actual: {feat_val:.1f}'
                ax_def_prox.text(bar.get_width() - 0.02, bar.get_y() + bar.get_height()/2,
                                label, ha='right', va='center', fontsize=8, fontweight='bold', color='black')
        except Exception as e:
            ax_def_prox.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center', fontsize=10)
            ax_def_prox.set_title('Local Prox Importance', fontsize=12, fontweight='bold')

        # =====================================================================
        # ROW 1: NON-DEFAULTER ANALYSIS - RFX vs SHAP + Top-K + Local Prox Imp
        # =====================================================================

        # Panel 1,0: Non-Defaulter Local Variable Importance (RFX vs SHAP with overlap)
        ax_nondef_shap = fig_b.add_subplot(gs[1, 0])

        sample_nondef = X_train[query_idx_nondef:query_idx_nondef+1]
        rfx_top6_nondef = set()
        shap_top6_nondef = set()

        if HAS_SHAP and HAS_XGBOOST:
            try:
                rfx_local_nondef = local_var[query_idx_nondef] if local_var is not None else None

                if rfx_local_nondef is not None:
                    rfx_local_abs_nondef = np.abs(rfx_local_nondef)
                    rfx_local_norm_nondef = rfx_local_abs_nondef / (rfx_local_abs_nondef.sum() + 1e-10)
                    rfx_top6_nondef = set(np.argsort(rfx_local_abs_nondef)[-6:])

                    shap_values_nondef = explainer.shap_values(sample_nondef)
                    shap_values_flat_nondef = shap_values_nondef[1] if isinstance(shap_values_nondef, list) else shap_values_nondef[0]
                    shap_abs_nondef = np.abs(shap_values_flat_nondef).flatten()
                    shap_norm_nondef = shap_abs_nondef / (shap_abs_nondef.sum() + 1e-10)
                    shap_top6_nondef = set(np.argsort(shap_abs_nondef)[-6:])

                    # Calculate overlap
                    overlap_nondef = len(rfx_top6_nondef & shap_top6_nondef)

                    combined_importance_nondef = rfx_local_norm_nondef + shap_norm_nondef
                    top_feat_indices_nondef = np.argsort(combined_importance_nondef)[-6:][::-1]

                    y_pos = np.arange(len(top_feat_indices_nondef))
                    width = 0.35

                    ax_nondef_shap.barh(y_pos - width/2, rfx_local_norm_nondef[top_feat_indices_nondef], width,
                                       label='RFX-Fuse', color='steelblue', alpha=0.8, edgecolor='navy')
                    ax_nondef_shap.barh(y_pos + width/2, shap_norm_nondef[top_feat_indices_nondef], width,
                                       label='SHAP', color='orange', alpha=0.8, edgecolor='darkorange')

                    ax_nondef_shap.set_xlabel('Normalized Importance', fontsize=10, fontweight='bold')
                    ax_nondef_shap.set_title(f'Why Predicted Safe?\n(RFX-Fuse vs SHAP, Top-6 Agreement: {overlap_nondef}/6)', fontsize=10, fontweight='bold')
                    ax_nondef_shap.set_yticks(y_pos)
                    ax_nondef_shap.set_yticklabels([feature_names[i][:15] for i in top_feat_indices_nondef], fontsize=9)
                    ax_nondef_shap.legend(fontsize=8)
                    ax_nondef_shap.grid(axis='x', alpha=0.3)
                    ax_nondef_shap.invert_yaxis()
            except Exception as e:
                ax_nondef_shap.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center', fontsize=10)
                ax_nondef_shap.set_title('Local Importance', fontsize=12, fontweight='bold')
        else:
            ax_nondef_shap.text(0.5, 0.5, 'SHAP not available', ha='center', va='center', fontsize=11)
            ax_nondef_shap.set_title('Local Importance', fontsize=12, fontweight='bold')

        # Panel 1,1: Non-Defaulter Top-K Similar
        ax_nondef_topk = fig_b.add_subplot(gs[1, 1])

        try:
            similar_indices_nondef, similar_scores_nondef = clf.get_top_k_similar(query_idx_nondef, k=TOP_K)

            similar_outcomes_nondef = []
            similar_risks_nondef = []
            for idx, score in zip(similar_indices_nondef, similar_scores_nondef):
                if idx < len(y_train):
                    risk = clf.predict_proba(X_train[idx:idx+1])[0]
                    risk_pct = (risk[1] if len(risk) > 1 else risk[0]) * 100
                    similar_outcomes_nondef.append(1 if y_train[idx] == 1 else 0)
                    similar_risks_nondef.append(risk_pct)

            if len(similar_risks_nondef) > 0:
                x_pos = np.arange(len(similar_risks_nondef))
                colors_nondef = ['salmon' if o == 1 else 'lightgreen' for o in similar_outcomes_nondef]
                ax_nondef_topk.bar(x_pos, similar_risks_nondef, color=colors_nondef, alpha=0.8, edgecolor='black', linewidth=1)

                ax_nondef_topk.axhline(50, color='red', linestyle='--', linewidth=2, label='50% threshold')
                ax_nondef_topk.set_xlabel('Similar Borrowers', fontsize=10, fontweight='bold')
                ax_nondef_topk.set_ylabel('Default Risk (%)', fontsize=10, fontweight='bold')

                n_paid_nondef = len(similar_outcomes_nondef) - sum(similar_outcomes_nondef)
                ax_nondef_topk.set_title(f'Neighbors of Non-Defaulter\n{n_paid_nondef}/{len(similar_outcomes_nondef)} also paid',
                                        fontsize=10, fontweight='bold')
                ax_nondef_topk.set_xticks(x_pos)
                ax_nondef_topk.set_xticklabels([f'#{i+1}' for i in range(len(similar_risks_nondef))], fontsize=8)
                ax_nondef_topk.legend(fontsize=8)
                ax_nondef_topk.grid(axis='y', alpha=0.3)
                ax_nondef_topk.set_ylim(0, 105)
        except Exception as e:
            ax_nondef_topk.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center', fontsize=10)
            ax_nondef_topk.set_title('Top-K Similar', fontsize=12, fontweight='bold')

        # Panel 1,2: Non-Defaulter Local Proximity Importance (WHY similar to neighbors)
        ax_nondef_prox = fig_b.add_subplot(gs[1, 2])

        try:
            local_prox_nondef = overall_prox_clf[query_idx_nondef]
            local_prox_sorted_nondef = np.argsort(local_prox_nondef)[::-1][:8]
            local_prox_norm_nondef = local_prox_nondef / (local_prox_nondef.max() + 1e-10)

            y_pos = np.arange(len(local_prox_sorted_nondef))
            prox_vals_nondef = local_prox_norm_nondef[local_prox_sorted_nondef]

            bars = ax_nondef_prox.barh(y_pos, prox_vals_nondef, color='mediumseagreen', alpha=0.8,
                                      edgecolor='darkgreen', linewidth=1.5)

            ax_nondef_prox.set_xlabel('Normalized Prox Importance', fontsize=10, fontweight='bold')
            ax_nondef_prox.set_title('Why Similar to Neighbors?\n(Local Prox Importance - RFX only)', fontsize=10, fontweight='bold')
            ax_nondef_prox.set_yticks(y_pos)
            ax_nondef_prox.set_yticklabels([feature_names[i][:15] for i in local_prox_sorted_nondef], fontsize=9)
            ax_nondef_prox.grid(axis='x', alpha=0.3)
            ax_nondef_prox.invert_yaxis()
            ax_nondef_prox.set_xlim(0, 1.05)

            # Add actual values inside bars (right-aligned, black text)
            for bar, feat_idx in zip(bars, local_prox_sorted_nondef):
                feat_val = X_train[query_idx_nondef, feat_idx]
                if feat_val == int(feat_val):
                    label = f'Actual: {int(feat_val)}'
                else:
                    label = f'Actual: {feat_val:.1f}'
                ax_nondef_prox.text(bar.get_width() - 0.02, bar.get_y() + bar.get_height()/2,
                                   label, ha='right', va='center', fontsize=8, fontweight='bold', color='black')
        except Exception as e:
            ax_nondef_prox.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center', fontsize=10)
            ax_nondef_prox.set_title('Local Prox Importance', fontsize=12, fontweight='bold')

        # =====================================================================
        # ROW 2: OUTLIER DETECTION - RFX-Fuse vs Isolation Forest
        # =====================================================================

        # Train Isolation Forest first (needed for multiple panels)
        # NOTE: Use training data (X_train, y_train) for consistency since outlier_scores
        # was computed on training data from unsup model
        try:
            iso_forest = IsolationForest(n_estimators=100, random_state=42, contamination='auto')
            iso_forest.fit(X_train)
            iso_scores = iso_forest.score_samples(X_train)

            # Normalize both scores for comparison (use classifier outlier scores for supervised context)
            # Use percentile-based clipping to handle extreme outliers that skew min-max
            # RFX: higher = more outlier (no negation needed)
            # IF: lower = more outlier (negate to match RFX convention)
            rfx_clip_max = np.percentile(outlier_scores_clf, 99)
            rfx_clipped = np.clip(outlier_scores_clf, outlier_scores_clf.min(), rfx_clip_max)
            rfx_norm = (rfx_clipped - rfx_clipped.min()) / (rfx_clipped.max() - rfx_clipped.min() + 1e-10)

            iso_negated = -iso_scores
            iso_clip_max = np.percentile(iso_negated, 99)
            iso_clipped = np.clip(iso_negated, iso_negated.min(), iso_clip_max)
            iso_norm = (iso_clipped - iso_clipped.min()) / (iso_clipped.max() - iso_clipped.min() + 1e-10)

            # Find top outlier (using classifier's outlier scores for consistency)
            # RFX: higher = more outlier, so use argmax
            top_outlier_idx = np.argmax(outlier_scores_clf)
            top_outlier_score = outlier_scores_clf[top_outlier_idx]
            top_outlier_actual = y_train[top_outlier_idx]
            top_outlier_outcome = 'DEFAULTED' if top_outlier_actual == 1 else 'Paid'
        except Exception as e:
            print(f"  Error computing outliers: {e}")
            import traceback
            traceback.print_exc()
            rfx_norm = np.zeros(len(y_train))
            iso_norm = np.zeros(len(y_train))
            top_outlier_idx = 0
            top_outlier_score = 0
            top_outlier_outcome = 'Unknown'

        # Panel 2,0: Outlier's Local Prox Imp (what makes this outlier unusual)
        ax_outlier_prox = fig_b.add_subplot(gs[2, 0])

        try:
            local_prox_outlier = overall_prox_clf[top_outlier_idx]
            local_prox_sorted_outlier = np.argsort(local_prox_outlier)[::-1][:8]
            local_prox_norm_outlier = local_prox_outlier / (local_prox_outlier.max() + 1e-10)

            labels_outlier = [feature_names[i][:12] for i in local_prox_sorted_outlier]
            actual_vals_outlier = [X_train[top_outlier_idx, i] for i in local_prox_sorted_outlier]
            vals_outlier = [local_prox_norm_outlier[i] for i in local_prox_sorted_outlier]
            y_outlier = np.arange(len(labels_outlier))

            bars_outlier = ax_outlier_prox.barh(y_outlier, vals_outlier, color='mediumseagreen', alpha=0.8,
                                                 edgecolor='darkgreen', linewidth=1.5)

            # Add actual values inside bars (right-aligned, black text)
            for bar, val in zip(bars_outlier, actual_vals_outlier):
                ax_outlier_prox.text(bar.get_width() - 0.02, bar.get_y() + bar.get_height()/2,
                                     f'Actual: {val:.1f}', va='center', ha='right', fontsize=8,
                                     fontweight='bold', color='black')

            ax_outlier_prox.set_xlabel('Prox Importance', fontsize=10, fontweight='bold')
            ax_outlier_prox.set_title(f'Top Outlier #{top_outlier_idx}: Local Prox Imp\n(RFX-Fuse capability only)',
                                      fontsize=11, fontweight='bold')
            ax_outlier_prox.set_yticks(y_outlier)
            ax_outlier_prox.set_yticklabels(labels_outlier, fontsize=9)
            ax_outlier_prox.grid(axis='x', alpha=0.3)
            ax_outlier_prox.invert_yaxis()
        except Exception as e:
            ax_outlier_prox.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center',
                                 fontsize=10, transform=ax_outlier_prox.transAxes)
            ax_outlier_prox.set_title('Outlier Local Prox Imp', fontsize=12, fontweight='bold')

        # Panel 2,1: RFX vs IF Outlier Score Distributions (like bike dataset)
        ax_outlier_dist = fig_b.add_subplot(gs[2, 1])

        try:
            # Overlay RFX and IF normalized score distributions
            ax_outlier_dist.hist(rfx_norm, bins=50, alpha=0.6, color='steelblue',
                                edgecolor='navy', label='RFX-Fuse (manifold)')
            ax_outlier_dist.hist(iso_norm, bins=50, alpha=0.5, color='orange',
                                edgecolor='darkorange', label='IF (flipped)')

            # Add 90th percentile thresholds
            rfx_threshold_90 = np.percentile(rfx_norm, 90)
            iso_threshold_90 = np.percentile(iso_norm, 90)
            ax_outlier_dist.axvline(rfx_threshold_90, color='steelblue', linestyle='--', linewidth=2.5,
                                   label=f'RFX 90th %ile ({rfx_threshold_90:.2f})')
            ax_outlier_dist.axvline(iso_threshold_90, color='orange', linestyle='--', linewidth=2.5,
                                   label=f'IF 90th %ile ({iso_threshold_90:.2f})')

            ax_outlier_dist.set_xlabel('Normalized Outlier Score', fontsize=10, fontweight='bold')
            ax_outlier_dist.set_ylabel('Count', fontsize=10, fontweight='bold')
            ax_outlier_dist.set_title('Outlier Score Distributions\n(IF flipped: higher=outlier)',
                                     fontsize=11, fontweight='bold')
            ax_outlier_dist.legend(fontsize=8, loc='upper right')
            ax_outlier_dist.grid(alpha=0.3)
        except Exception as e:
            ax_outlier_dist.text(0.5, 0.5, f'Error:\n{str(e)[:30]}', ha='center', va='center',
                                fontsize=10, transform=ax_outlier_dist.transAxes)
            ax_outlier_dist.set_title('Outlier Distribution', fontsize=12, fontweight='bold')

        # Panel 2,2: RFX vs IF Scatterplot (like bike dataset)
        ax_if = fig_b.add_subplot(gs[2, 2])

        try:
            # 90th percentile thresholds
            rfx_threshold = np.percentile(rfx_norm, 90)
            iso_threshold = np.percentile(iso_norm, 90)

            # Color code points by agreement
            rfx_outlier_mask = rfx_norm > rfx_threshold
            iso_outlier_mask = iso_norm > iso_threshold

            # Color: gray=neither, blue=RFX only, orange=IF only, purple=both
            colors = np.array(['lightgray'] * len(rfx_norm))
            colors[rfx_outlier_mask & ~iso_outlier_mask] = 'steelblue'
            colors[~rfx_outlier_mask & iso_outlier_mask] = 'orange'
            colors[rfx_outlier_mask & iso_outlier_mask] = 'purple'

            ax_if.scatter(rfx_norm, iso_norm, alpha=0.5, s=12, c=colors, edgecolors='none')

            # Highlight top outlier with a star marker
            ax_if.scatter(rfx_norm[top_outlier_idx], iso_norm[top_outlier_idx],
                         s=300, marker='*', c='red', edgecolors='darkred', linewidths=1.5,
                         label=f'Top Outlier #{top_outlier_idx}', zorder=10)

            # Add diagonal line
            ax_if.plot([0, 1], [0, 1], 'k--', linewidth=1.5, alpha=0.5, label='Perfect Agreement')

            # Add threshold lines matching color scheme
            ax_if.axvline(rfx_threshold, color='steelblue', linestyle='--', linewidth=2, alpha=0.7, label=f'RFX-Fuse 90th: {rfx_threshold:.2f}')
            ax_if.axhline(iso_threshold, color='orange', linestyle='--', linewidth=2, alpha=0.7, label=f'IF 90th: {iso_threshold:.2f}')

            # Correlation and overlap
            corr_outlier = np.corrcoef(rfx_norm, iso_norm)[0, 1]
            rfx_top10 = set(np.where(rfx_outlier_mask)[0])
            iso_top10 = set(np.where(iso_outlier_mask)[0])
            overlap = len(rfx_top10 & iso_top10)
            overlap_pct = 100 * overlap / len(rfx_top10) if len(rfx_top10) > 0 else 0

            ax_if.set_xlabel('RFX-Fuse Outlier Score', fontsize=10, fontweight='bold', color='steelblue')
            ax_if.set_ylabel('IF Outlier Score (flipped)', fontsize=10, fontweight='bold', color='orange')
            ax_if.set_title(f'RFX-Fuse vs IF (IF flipped: higher=outlier)\nCorr: {corr_outlier:.3f}, Overlap: {overlap_pct:.0f}%', fontsize=11, fontweight='bold')
            ax_if.legend(fontsize=7, loc='lower right')
            ax_if.grid(alpha=0.3)
            ax_if.set_xlim(-0.05, 1.05)
            ax_if.set_ylim(-0.05, 1.05)
        except Exception as e:
            ax_if.text(0.5, 0.5, f'IF error:\n{str(e)[:30]}', ha='center', va='center',
                      fontsize=10, transform=ax_if.transAxes)
            ax_if.set_title('RFX-Fuse vs Isolation Forest', fontsize=12, fontweight='bold')

        # Save Figure B
        fig_b_path = SCRIPT_DIR / "loan_classification_9panel_b.png"
        fig_b_path.parent.mkdir(parents=True, exist_ok=True)
        plt.savefig(fig_b_path, dpi=300, bbox_inches='tight')
        print(f"\nSaved Figure B (Explainability Stories) to: {fig_b_path}")
        plt.close(fig_b)

    except Exception as e:
        print(f"\n  Could not create Figure B: {e}")
        import traceback
        traceback.print_exc()
    
    print(f"\n{'='*75}")
    print("SUMMARY: RFX-Fuse (1 Model) vs Traditional 3-Tool Approach")
    print("=" * 75)
    print(f"""
Traditional Approach: XGBoost + SHAP + Isolation Forest
  • 3 separate tools to configure & orchestrate
  • Post-hoc explanations (SHAP - slow on large data)
  • Separate outlier detection (Isolation Forest)
  • No unified similarity explanations

RFX Approach: One model object does it all
  • Classification (comparable to XGBoost)
  • Local importance (comparable to SHAP)
  • Outlier detection (complementary to Isolation Forest - marginal vs. manifold)
  • Similarity search with explanations
  • OOB error estimation
  • Proximity importance
""")
    
    # REGULATORY SUMMARY
    # =========================================================================
    print(f"\n{'='*75}")
    print("REGULATORY COMPLIANCE SUMMARY")
    print("=" * 75)
    
    print(f"""
+-------------------------------------------------------------------------+
|                    RFX FOR REGULATORY COMPLIANCE                        |
+-------------------------------------------------------------------------+
| ECOA Adverse Action Notices:                                            |
|   * Local Variable Importance provides principal denial reasons         |
|   * "Your application was denied due to: Outstanding Debt (+0.02),     |
|      Delayed Payments (+0.01), Credit Utilization (+0.01)"             |
+-------------------------------------------------------------------------+
| Fair Lending Documentation:                                             |
|   * Overall Variable Importance shows global decision factors           |
|   * Demonstrates consistent, explainable decision criteria              |
|   * Auditable: "The model primarily uses {feature_names[sorted_var[0]]}"        |
+-------------------------------------------------------------------------+
| GDPR Article 22 (Right to Explanation):                                 |
|   * Local Importance: "Why was MY application denied?"                  |
|   * Proximity Importance: "What borrowers am I similar to?"             |
|   * Complete audit trail for each decision                              |
+-------------------------------------------------------------------------+

Training Times:
  * Classifier:    {clf_time:.1f}s

Outlier Detection (On-Demand):
  * Classifier outliers: {outlier_time_clf:.2f}s (greedy mode, using leaf assignments)
  * No full proximity matrix needed - memory efficient!

Key Insight:
  RFX provides COMPLETE explainability for credit decisions:
  - Global model behavior (Overall Variable Importance)
  - Individual decision reasons (Local Variable Importance)  
  - Borrower similarity context (Proximity Importance)
  - Outlier detection (On-demand, using leaf assignments - no 5-hour wait!)
  - All native—no post-hoc SHAP required
""")


if __name__ == "__main__":
    main()

