
import pandas as pd
import numpy as np
import joblib
import json
from sklearn.model_selection import train_test_split
from datetime import datetime

from .config import Config
from .eda import FeatureAnalyzer
from .binning import BinningProcess
from .selection import FeatureSelector
from .modeling import ModelTrainer
from .evaluation import ModelEvaluator
from .scorecard import ScorecardTransformer
from .reporting import ReportGenerator
from .plot_utils import plot_binning

class AutoScore:
    def __init__(self, random_state=42):
        self.random_state = random_state
        self.eda = FeatureAnalyzer()
        self.binning = None
        self.selector = FeatureSelector()
        self.trainer = ModelTrainer(random_state=random_state)
        self.evaluator = ModelEvaluator()
        self.sc_transformer = None
        self.reporter = ReportGenerator()
        
        # Intermediate results
        self.data_summary = None
        self.iv_df = None
        self.selected_features = []
        self.model_metrics = {}
        self.scorecard_df = None
        
    @staticmethod
    def validate_date_format(date_str, expected_format='yyyy-mm-dd'):
        """
        Validate date format and convert to standard format.
        
        Parameters:
        -----------
        date_str : str
            Date string to validate
        expected_format : str
            Expected format description (default: 'yyyy-mm-dd')
            
        Returns:
        --------
        str
            Date in 'yyyy-mm-dd' format
            
        Raises:
        -------
        ValueError
            If date format is invalid or cannot be parsed
        """
        if pd.isna(date_str):
            raise ValueError(f"Date value is None/NaN. Expected format: {expected_format}")
        
        if not isinstance(date_str, str):
            try:
                date_str = str(date_str)
            except:
                raise ValueError(f"Cannot convert date to string. Expected format: {expected_format}")
        
        # Try to parse the date using pandas
        try:
            parsed_date = pd.to_datetime(date_str, errors='coerce')
            if pd.isna(parsed_date):
                raise ValueError(f"Invalid date format: '{date_str}'. Expected format: {expected_format}")
            # Return in standard format
            return parsed_date.strftime('%Y-%m-%d')
        except Exception as e:
            raise ValueError(f"Date parsing error for '{date_str}': {str(e)}. Expected format: {expected_format}")
    
    @staticmethod
    def convert_date_column(date_series):
        """
        Convert date column to standard 'yyyy-mm-dd' format.
        Automatically detects and converts various date formats.
        
        Parameters:
        -----------
        date_series : pd.Series
            Date column to convert
            
        Returns:
        --------
        pd.Series
            Date column in 'yyyy-mm-dd' format
            
        Raises:
        -------
        ValueError
            If conversion fails for most values
        """
        # Try to convert using pandas to_datetime
        try:
            converted = pd.to_datetime(date_series, errors='coerce')
            
            # Check conversion success rate
            success_rate = converted.notna().mean()
            
            if success_rate < 0.8:
                raise ValueError(f"Date conversion failed for {(1-success_rate)*100:.1f}% of values. Please check date format.")
            
            # Convert to standard string format
            return converted.dt.strftime('%Y-%m-%d')
            
        except Exception as e:
            raise ValueError(f"Failed to convert date column: {str(e)}")
        
    def fit(self, X, y, 
            test_size=0.3,
            date_col='create_time',
            oot_split_date=None,
            excluded_features=None,
            special_values=None,
            monotonicity_constraints=None,
            max_bins=10,
            min_bin_pct=0.05,
            iv_threshold=0.02,
            selection_method='stepwise',
            pdo=20, base_score=600, base_odds=50,
            custom_binning_rules=None,
            final_bins=None,
            final_features=None):
        """
        Execute full scorecard development pipeline with support for intermediate process inputs.
        
        Parameters:
        -----------
        X : pd.DataFrame
            Feature matrix
        y : pd.Series
            Target variable (0/1, 1 represents bad customer)
        test_size : float, default=0.3
            Test set ratio
        date_col : str, default='create_time'
            Date column name for OOT split
        oot_split_date : str, optional
            OOT split date in 'yyyy-mm-dd' format
        excluded_features : list, optional
            Features to exclude from modeling
        special_values : list, optional
            Special values to treat as separate bins (e.g., -999)
        monotonicity_constraints : dict, optional
            Monotonicity constraints: {feature: 1|-1|0}
            1: increasing, -1: decreasing, 0: no constraint
        max_bins : int, default=10
            Maximum number of bins
        min_bin_pct : float, default=0.05
            Minimum bin percentage
        iv_threshold : float, default=0.02
            IV threshold for feature filtering
        selection_method : str, default='stepwise'
            Feature selection method: 'stepwise' or 'iv'
        pdo : int, default=20
            Points to Double the Odds
        base_score : int, default=600
            Base score
        base_odds : int, default=50
            Base odds at base_score
        custom_binning_rules : dict, optional
            Custom binning rules (same format as final_bins)
        final_bins : dict, optional
            Final binning rules to skip binning step
            Format: {feature: {'type': 'numeric'|'categorical', 'cutoffs': [...], 'categories': [...], 'woe_map': {...}}}
        final_features : list, optional
            Final selected features to skip feature selection step
            
        Returns:
        --------
        dict
            Dictionary containing all intermediate results:
            - numeric_features: List of numeric features
            - categorical_features: List of categorical features
            - converted_cols: List of automatically converted columns
            - selected_features: Final selected features
            - monotonicity_constraints: Applied monotonicity constraints
            - binning_rules: Final binning rules
            - woe_transformer: Binning process object
            - model_trainer: Model trainer object
            - model: Trained model
            - scorecard_transformer: Scorecard transformer object
            - performance: Model performance metrics
            - scorecard_df: Scorecard DataFrame
        """
        # Preprocessing: Exclude features
        if excluded_features:
            X = X.drop(columns=[c for c in excluded_features if c in X.columns])
            
        # 0. Smart Type Detection
        X, converted_cols = self.eda.detect_and_convert_types(X)
        if converted_cols:
            print(f"Automatically converted to numeric: {converted_cols}")
            
        # 1. Split Data (Train/Test/OOT)
        X_oot, y_oot = None, None
        
        if date_col in X.columns and oot_split_date:
            # Validate and convert oot_split_date to standard format
            try:
                oot_split_date_std = self.validate_date_format(oot_split_date)
                print(f"OOT split date validated: {oot_split_date_std}")
            except ValueError as e:
                print(f"ERROR: {e}")
                raise
            
            # Convert date column to standard format
            try:
                print(f"Converting date column '{date_col}' to standard format...")
                X[date_col] = self.convert_date_column(X[date_col])
                print(f"Date column conversion completed successfully")
            except ValueError as e:
                print(f"ERROR: {e}")
                raise
            
            print(f"Splitting OOT data based on {date_col} > {oot_split_date_std}")
            mask_oot = X[date_col] > oot_split_date_std
            X_oot = X[mask_oot].copy()
            y_oot = y[mask_oot].copy()
            X = X[~mask_oot].copy()
            y = y[~mask_oot].copy()
            
            # Drop date column from modeling
            X_oot = X_oot.drop(columns=[date_col])
            X = X.drop(columns=[date_col])
        elif date_col in X.columns:
            # Just convert date column format even without OOT split
            try:
                print(f"Converting date column '{date_col}' to standard format...")
                X[date_col] = self.convert_date_column(X[date_col])
                print(f"Date column conversion completed successfully")
            except ValueError as e:
                print(f"WARNING: {e}")
                print(f"Proceeding without date conversion...")
            X = X.drop(columns=[date_col])
        
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=test_size, random_state=self.random_state, stratify=y
        )
        
        # 收集数据集统计信息
        self.dataset_stats = {
            'original_total': len(X),
            'filtered_total': len(X),
            'train': {
                'total': len(X_train),
                'good': (y_train == 0).sum(),
                'bad': (y_train == 1).sum(),
                'bad_rate': (y_train == 1).mean()
            },
            'test': {
                'total': len(X_test),
                'good': (y_test == 0).sum(),
                'bad': (y_test == 1).sum(),
                'bad_rate': (y_test == 1).mean()
            }
        }
        
        if X_oot is not None:
            self.dataset_stats['oot'] = {
                'total': len(X_oot),
                'good': (y_oot == 0).sum(),
                'bad': (y_oot == 1).sum(),
                'bad_rate': (y_oot == 1).mean()
            }
        else:
            self.dataset_stats['oot'] = None
        
        # 2. EDA
        print("Running EDA...")
        self.data_summary = self.eda.analyze(X_train)
        
        # 3. Binning
        print("Running Binning...")
        self.binning = BinningProcess(max_bins=max_bins, min_bin_pct=min_bin_pct, special_values=special_values)
        
        if final_bins:
            print(f"Using provided final binning rules ({len(final_bins)} features)...")
            print("Skipping automatic binning step.")
            self.binning.set_rules(final_bins)
            # Recalculate stats for the current training data using these rules
            self.binning.recalculate_stats(X_train, y_train)
        elif custom_binning_rules:
            print("Using custom binning rules...")
            self.binning.set_rules(custom_binning_rules)
            # Recalculate stats for the current training data using these rules
            self.binning.recalculate_stats(X_train, y_train)
        else:
            self.binning.fit(X_train, y_train, monotonicity_constraints=monotonicity_constraints)
        
        self.iv_df = self.binning.iv_df
        
        # Transform to WOE
        X_train_woe = self.binning.transform(X_train)
        X_test_woe = self.binning.transform(X_test)
        X_oot_woe = self.binning.transform(X_oot) if X_oot is not None else None
        
        # 4. Feature Selection
        print("Running Feature Selection...")
        if final_features:
             print(f"Using provided final features ({len(final_features)})...")
             print("Skipping feature selection step.")
             self.selected_features = final_features
             # Validate that all final_features exist in X_train_woe
             missing_features = [f for f in final_features if f not in X_train_woe.columns]
             if missing_features:
                 raise ValueError(f"Final features not found in data: {missing_features}")
        else:
            # IV Filter
            features_iv = self.selector.filter_by_iv(self.iv_df, threshold=iv_threshold)
            print(f"Features after IV filter: {len(features_iv)}")
            
            # Stepwise
            if selection_method == 'stepwise':
                self.selected_features = self.selector.stepwise_selection(
                    X_train_woe[features_iv], y_train, method='forward'
                )
            else:
                self.selected_features = features_iv
            
        print(f"Final selected features: {len(self.selected_features)}")
        print(self.selected_features)
        
        # 5. Model Training
        print("Training Model...")
        self.trainer.train(X_train_woe[self.selected_features], y_train)
        
        # 6. Evaluation
        print("Evaluating Model...")
        self.evaluate_performance(X_train_woe, y_train, X_test_woe, y_test, X_oot_woe, y_oot)
        
        # 7. Scorecard Generation
        print("Generating Scorecard...")
        self.sc_transformer = ScorecardTransformer(pdo=pdo, base_score=base_score, base_odds=base_odds)
        self.scorecard_df = self.sc_transformer.fit(self.trainer.model, self.binning, self.selected_features)
        
        # 收集各数据集的分箱统计
        self.binning_stats = self._collect_binning_stats(X_train, y_train, X_test, y_test, X_oot, y_oot)
        
        # 收集各数据集的评分统计
        self.score_stats = self._collect_score_stats(X_train_woe, y_train, X_test_woe, y_test, X_oot_woe, y_oot)
        
        # Prepare Return Object with comprehensive intermediate results
        fit_result = {
            'numeric_features': [c for c in X.columns if pd.api.types.is_numeric_dtype(X[c])],
            'categorical_features': [c for c in X.columns if not pd.api.types.is_numeric_dtype(X[c])],
            'converted_cols': converted_cols,
            'selected_features': self.selected_features,
            'monotonicity_constraints': monotonicity_constraints,
            'binning_rules': self.binning.rules,
            'woe_transformer': self.binning,
            'model_trainer': self.trainer,
            'model': self.trainer.model,
            'scorecard_transformer': self.sc_transformer,
            'performance': self.model_metrics,
            'scorecard_df': self.scorecard_df,
            'iv_df': self.iv_df,
            'data_summary': self.data_summary
        }
        
        return fit_result

    def evaluate_performance(self, X_train_woe, y_train, X_test_woe, y_test, X_oot_woe=None, y_oot=None):
        """
        Evaluate model on Train/Test/OOT.
        """
        # Handle NaN values in WOE data
        X_train_clean = X_train_woe[self.selected_features].fillna(0)
        X_test_clean = X_test_woe[self.selected_features].fillna(0)
        
        # Train metrics
        pred_train = self.trainer.predict_proba(X_train_clean)
        metrics_train = self.evaluator.evaluate(y_train, pred_train)
        
        # Test metrics
        pred_test = self.trainer.predict_proba(X_test_clean)
        metrics_test = self.evaluator.evaluate(y_test, pred_test)
        
        # PSI (Train vs Test)
        psi_test = self.evaluator.calculate_psi(pred_train, pred_test)
        
        self.model_metrics = {
            'train': metrics_train,
            'test': metrics_test,
            'psi_train_test': psi_test
        }
        
        if X_oot_woe is not None:
            X_oot_clean = X_oot_woe[self.selected_features].fillna(0)
            pred_oot = self.trainer.predict_proba(X_oot_clean)
            metrics_oot = self.evaluator.evaluate(y_oot, pred_oot)
            psi_oot = self.evaluator.calculate_psi(pred_train, pred_oot)
            self.model_metrics['oot'] = metrics_oot
            self.model_metrics['psi_train_oot'] = psi_oot
    
    def _collect_binning_stats(self, X_train, y_train, X_test, y_test, X_oot=None, y_oot=None):
        """
        Collect binning statistics for each dataset.
        """
        binning_stats = {}
        
        for dataset_name, X, y in [('train', X_train, y_train), ('test', X_test, y_test)]:
            stats_list = []
            for feat in self.selected_features:
                if feat not in self.binning.rules:
                    continue
                
                rule = self.binning.rules[feat]
                
                if rule['type'] == 'numeric':
                    # Numeric feature binning
                    cutoffs = rule['cutoffs']
                    bins = [-np.inf] + cutoffs + [np.inf]
                    
                    # Assign bins
                    X_copy = X.copy()
                    x_col = pd.to_numeric(X_copy[feat], errors='coerce')
                    bin_indices = pd.cut(x_col, bins=bins, labels=False, include_lowest=True)
                    
                    # Calculate statistics for each bin
                    for bin_idx in range(len(bins) - 1):
                        mask = bin_indices == bin_idx
                        bin_good = (y[mask] == 0).sum()
                        bin_bad = (y[mask] == 1).sum()
                        bin_total = bin_good + bin_bad
                        bin_bad_rate = bin_bad / bin_total if bin_total > 0 else 0
                        
                        stats_list.append({
                            'feature': feat,
                            'dataset': dataset_name,
                            'bin': bin_idx,
                            'bin_range': f'[{bins[bin_idx]}, {bins[bin_idx+1]})',
                            'count': bin_total,
                            'good': bin_good,
                            'bad': bin_bad,
                            'bad_rate': bin_bad_rate
                        })
                else:
                    # Categorical feature binning
                    categories = rule['categories']
                    
                    for bin_idx, bin_vals in enumerate(categories):
                        mask = X[feat].isin(bin_vals)
                        bin_good = (y[mask] == 0).sum()
                        bin_bad = (y[mask] == 1).sum()
                        bin_total = bin_good + bin_bad
                        bin_bad_rate = bin_bad / bin_total if bin_total > 0 else 0
                        
                        stats_list.append({
                            'feature': feat,
                            'dataset': dataset_name,
                            'bin': bin_idx,
                            'bin_range': str(bin_vals),
                            'count': bin_total,
                            'good': bin_good,
                            'bad': bin_bad,
                            'bad_rate': bin_bad_rate
                        })
            
            binning_stats[dataset_name] = pd.DataFrame(stats_list)
        
        # OOT dataset
        if X_oot is not None:
            stats_list = []
            for feat in self.selected_features:
                if feat not in self.binning.rules:
                    continue
                
                rule = self.binning.rules[feat]
                
                if rule['type'] == 'numeric':
                    cutoffs = rule['cutoffs']
                    bins = [-np.inf] + cutoffs + [np.inf]
                    
                    X_copy = X_oot.copy()
                    x_col = pd.to_numeric(X_copy[feat], errors='coerce')
                    bin_indices = pd.cut(x_col, bins=bins, labels=False, include_lowest=True)
                    
                    for bin_idx in range(len(bins) - 1):
                        mask = bin_indices == bin_idx
                        bin_good = (y_oot[mask] == 0).sum()
                        bin_bad = (y_oot[mask] == 1).sum()
                        bin_total = bin_good + bin_bad
                        bin_bad_rate = bin_bad / bin_total if bin_total > 0 else 0
                        
                        stats_list.append({
                            'feature': feat,
                            'dataset': 'oot',
                            'bin': bin_idx,
                            'bin_range': f'[{bins[bin_idx]}, {bins[bin_idx+1]})',
                            'count': bin_total,
                            'good': bin_good,
                            'bad': bin_bad,
                            'bad_rate': bin_bad_rate
                        })
                else:
                    categories = rule['categories']
                    
                    for bin_idx, bin_vals in enumerate(categories):
                        mask = X_oot[feat].isin(bin_vals)
                        bin_good = (y_oot[mask] == 0).sum()
                        bin_bad = (y_oot[mask] == 1).sum()
                        bin_total = bin_good + bin_bad
                        bin_bad_rate = bin_bad / bin_total if bin_total > 0 else 0
                        
                        stats_list.append({
                            'feature': feat,
                            'dataset': 'oot',
                            'bin': bin_idx,
                            'bin_range': str(bin_vals),
                            'count': bin_total,
                            'good': bin_good,
                            'bad': bin_bad,
                            'bad_rate': bin_bad_rate
                        })
            
            binning_stats['oot'] = pd.DataFrame(stats_list)
        
        return binning_stats
    
    def _collect_score_stats(self, X_train_woe, y_train, X_test_woe, y_test, X_oot_woe=None, y_oot=None):
        """
        Collect score distribution statistics for each dataset.
        """
        score_stats = {}
        
        for dataset_name, X_woe, y in [('train', X_train_woe, y_train), ('test', X_test_woe, y_test)]:
            # Calculate scores
            X_sel = X_woe[self.selected_features]
            scores = self.sc_transformer.transform(X_sel, self.selected_features, self.trainer.model)
            
            # Equal frequency binning (20 bins)
            try:
                score_bins = pd.qcut(scores, q=20, labels=False, duplicates='drop')
            except ValueError:
                # If not enough unique values, use cut instead
                score_bins = pd.cut(scores, bins=20, labels=False, include_lowest=True)
            
            # Calculate statistics for each score bin
            stats_list = []
            unique_bins = sorted(np.unique(score_bins)) if isinstance(score_bins, np.ndarray) else sorted(score_bins.unique())
            for bin_idx in unique_bins:
                mask = score_bins == bin_idx
                bin_good = (y[mask] == 0).sum()
                bin_bad = (y[mask] == 1).sum()
                bin_total = bin_good + bin_bad
                bin_bad_rate = bin_bad / bin_total if bin_total > 0 else 0
                
                # Cumulative rates
                cum_good = (y == 0).cumsum()[mask].max()
                cum_bad = (y == 1).cumsum()[mask].max()
                total_good = (y == 0).sum()
                total_bad = (y == 1).sum()
                cum_good_rate = cum_good / total_good if total_good > 0 else 0
                cum_bad_rate = cum_bad / total_bad if total_bad > 0 else 0
                
                stats_list.append({
                    'dataset': dataset_name,
                    'bin': bin_idx,
                    'score_range': f'Bin {bin_idx+1}',
                    'count': bin_total,
                    'good': bin_good,
                    'bad': bin_bad,
                    'bad_rate': bin_bad_rate,
                    'cum_bad_rate': cum_bad_rate,
                    'cum_good_rate': cum_good_rate
                })
            
            score_stats[dataset_name] = pd.DataFrame(stats_list)
        
        # OOT dataset
        if X_oot_woe is not None:
            X_sel = X_oot_woe[self.selected_features]
            scores = self.sc_transformer.transform(X_sel, self.selected_features, self.trainer.model)
            
            # Equal frequency binning (20 bins)
            try:
                score_bins = pd.qcut(scores, q=20, labels=False, duplicates='drop')
            except ValueError:
                score_bins = pd.cut(scores, bins=20, labels=False, include_lowest=True)
            
            # Calculate statistics for each score bin
            stats_list = []
            unique_bins = sorted(np.unique(score_bins)) if isinstance(score_bins, np.ndarray) else sorted(score_bins.unique())
            for bin_idx in unique_bins:
                mask = score_bins == bin_idx
                bin_good = (y_oot[mask] == 0).sum()
                bin_bad = (y_oot[mask] == 1).sum()
                bin_total = bin_good + bin_bad
                bin_bad_rate = bin_bad / bin_total if bin_total > 0 else 0
                
                # Cumulative rates
                cum_good = (y_oot == 0).cumsum()[mask].max()
                cum_bad = (y_oot == 1).cumsum()[mask].max()
                total_good = (y_oot == 0).sum()
                total_bad = (y_oot == 1).sum()
                cum_good_rate = cum_good / total_good if total_good > 0 else 0
                cum_bad_rate = cum_bad / total_bad if total_bad > 0 else 0
                
                stats_list.append({
                    'dataset': 'oot',
                    'bin': bin_idx,
                    'score_range': f'Bin {bin_idx+1}',
                    'count': bin_total,
                    'good': bin_good,
                    'bad': bin_bad,
                    'bad_rate': bin_bad_rate,
                    'cum_bad_rate': cum_bad_rate,
                    'cum_good_rate': cum_good_rate
                })
            
            score_stats['oot'] = pd.DataFrame(stats_list)
        
        return score_stats

    def update_model_weights(self, coef_dict, intercept=None):
        """
        Manually update model weights and re-evaluate.
        coef_dict: {feature: weight}
        """
        if self.trainer.model is None:
            print("Model not trained yet.")
            return
            
        # Get estimator
        estimator = self.trainer.model.best_estimator_ if hasattr(self.trainer.model, 'best_estimator_') else self.trainer.model
        
        # Update coefficients
        # Assuming coef_dict contains all selected features
        new_coefs = []
        for feat in self.selected_features:
            if feat in coef_dict:
                new_coefs.append(coef_dict[feat])
            else:
                # Keep original? Or error?
                # Let's try to find index
                idx = self.selected_features.index(feat)
                new_coefs.append(estimator.coef_[0][idx])
                
        estimator.coef_ = np.array([new_coefs])
        
        if intercept is not None:
            estimator.intercept_ = np.array([intercept])
            
        print("Model weights updated.")
        # Note: Should re-run evaluation and scorecard generation ideally
        # But user needs to provide data for evaluation. 
        # So usually this is done before final save.
        
    def adjust_rules(self, new_rules, X_train=None, y_train=None):
        """
        Manually adjust binning rules and visualize.
        """
        if self.binning is None:
            print("Binning process not initialized. Run fit() first or provide custom rules.")
            return
            
        print("Adjusting rules...")
        # Update specific rules
        for col, rule in new_rules.items():
            if col in self.binning.rules:
                # Assuming user provides 'cutoffs' or 'categories' directly
                if 'cutoffs' in rule:
                    self.binning.rules[col]['cutoffs'] = rule['cutoffs']
                if 'categories' in rule:
                    self.binning.rules[col]['categories'] = rule['categories']
            else:
                print(f"Warning: {col} not found in existing rules.")
                
        # Recalculate stats if data provided (highly recommended)
        if X_train is not None and y_train is not None:
            self.binning.recalculate_stats(X_train, y_train)
            self.iv_df = self.binning.iv_df
            
    def visualize_binning(self, feature_name, filename=None, show=True):
        """
        Visualize binning for a feature.
        """
        if self.binning is None or feature_name not in self.binning.rules:
            print(f"No binning info for {feature_name}")
            return
            
        stats = self.binning.rules[feature_name]['stats']
        plot_binning(stats, feature_name, show=show, filename=filename)

    def save_model(self, filepath):
        """Save the entire AutoScore object."""
        joblib.dump(self, filepath)
        
    def load_model(self, filepath):
        """Load AutoScore object."""
        return joblib.load(filepath)
    
    def export_lightweight(self, output_path):
        """
        Export lightweight model for deployment without autoscore dependency.
        
        This method exports only the essential components needed for scoring:
        - Model coefficients and intercept
        - Binning rules (cutoffs, categories, WOE mappings)
        - Scoring parameters (factor, offset)
        - Selected features
        
        The exported model can be loaded using LightweightScorer class,
        which only requires numpy and pandas.
        
        Parameters:
        -----------
        output_path : str
            Path to save the lightweight model JSON file
            
        Returns:
        --------
        str
            Path to the exported model file
            
        Raises:
        -------
        ValueError
            If model has not been trained yet
            
        Example:
        --------
        >>> score = AutoScore()
        >>> score.fit(X, y)
        >>> score.export_lightweight('lightweight_model.json')
        >>> 
        >>> # In deployment environment (no autoscore needed):
        >>> from autoscore import LightweightScorer
        >>> scorer = LightweightScorer('lightweight_model.json')
        >>> scores = scorer.predict_score(X_new)
        """
        if self.trainer.model is None:
            raise ValueError("Model not trained yet. Please call fit() first.")
        
        if self.binning is None:
            raise ValueError("Binning not initialized. Please call fit() first.")
        
        if self.sc_transformer is None:
            raise ValueError("Scorecard not generated. Please call fit() first.")
        
        # 1. Extract model coefficients and intercept
        model = self.trainer.model
        if hasattr(model, 'best_estimator_'):
            estimator = model.best_estimator_
        else:
            estimator = model
        
        coefs = [float(c) for c in estimator.coef_[0]]
        intercept = float(estimator.intercept_[0])
        
        # 2. Extract binning rules
        binning_rules = {}
        for feat, rule in self.binning.rules.items():
            # Convert numpy types to Python native types
            woe_map_converted = {}
            for k, v in rule['woe_map'].items():
                woe_map_converted[str(k)] = float(v)
            
            stats_converted = []
            for stat in rule['stats']:
                # Handle both numeric and categorical features
                if rule['type'] == 'numeric':
                    min_val = float(stat.get('min', stat.get('lower', 0)))
                    max_val = float(stat.get('max', stat.get('upper', 0)))
                else:
                    # For categorical features, min/max are lists of categories
                    min_val = stat.get('min', stat.get('lower', []))
                    max_val = stat.get('max', stat.get('upper', []))
                
                stat_converted = {
                    'min': min_val,
                    'max': max_val,
                    'good': int(stat['good']),
                    'bad': int(stat['bad']),
                    'woe': float(stat['woe']),
                    'iv': float(stat.get('iv', 0))
                }
                stats_converted.append(stat_converted)
            
            binning_rules[feat] = {
                'type': rule['type'],
                'cutoffs': [float(c) for c in rule.get('cutoffs', [])],
                'categories': rule.get('categories', []),
                'woe_map': woe_map_converted,
                'stats': stats_converted
            }
        
        # 3. Extract scoring parameters
        scoring_params = {
            'pdo': int(self.sc_transformer.pdo),
            'base_score': int(self.sc_transformer.base_score),
            'base_odds': int(self.sc_transformer.base_odds),
            'factor': float(self.sc_transformer.factor),
            'offset': float(self.sc_transformer.offset),
            'selected_features': self.selected_features
        }
        
        # 4. Prepare model data
        model_data = {
            'metadata': {
                'version': '1.0.0',
                'created_at': pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S'),
                'description': 'Lightweight model exported from AutoScore'
            },
            'model': {
                'coefs': coefs,
                'intercept': intercept,
                'n_features': len(self.selected_features)
            },
            'binning_rules': binning_rules,
            'scoring_params': scoring_params
        }
        
        # 5. Save to JSON file
        with open(output_path, 'w', encoding='utf-8') as f:
            json.dump(model_data, f, indent=2, ensure_ascii=False)
        
        print(f"Lightweight model exported to: {output_path}")
        print(f"Model info:")
        print(f"  - Version: {model_data['metadata']['version']}")
        print(f"  - Features: {len(self.selected_features)}")
        print(f"  - PDO: {scoring_params['pdo']}")
        print(f"  - Base Score: {scoring_params['base_score']}")
        print(f"  - Base Odds: {scoring_params['base_odds']}")
        
        return output_path
    
    def online_fit(self, X_new, y_new, retain_binning=True, retain_features=True):
        """
        Online learning: Update model with new data.
        
        This method supports incremental learning scenarios where new data arrives
        over time. It can either:
        1. Retain existing binning rules and features (recommended for stability)
        2. Re-run full pipeline with new data
        
        Parameters:
        -----------
        X_new : pd.DataFrame
            New feature data
        y_new : pd.Series
            New target data
        retain_binning : bool, default=True
            If True, keep existing binning rules and only update model coefficients.
            If False, re-run binning on combined data.
        retain_features : bool, default=True
            If True, keep existing feature selection.
            If False, re-run feature selection.
            
        Returns:
        --------
        dict
            Updated performance metrics
            
        Raises:
        -------
        ValueError
            If model has not been trained yet
            
        Example:
        --------
        >>> # Initial training
        >>> score = AutoScore()
        >>> score.fit(X_train, y_train)
        >>> 
        >>> # Online learning with new data
        >>> new_metrics = score.online_fit(X_new, y_new)
        >>> print(f"Updated Test AUC: {new_metrics['test']['auc']}")
        """
        if self.trainer.model is None:
            raise ValueError("Model not trained yet. Please call fit() first.")
        
        if self.binning is None:
            raise ValueError("Binning not initialized. Please call fit() first.")
        
        print("Starting online learning...")
        
        if retain_binning:
            print("Retaining existing binning rules...")
            # Transform new data using existing binning rules
            X_new_woe = self.binning.transform(X_new)
            
            if retain_features:
                print("Retaining existing feature selection...")
                # Use existing features
                X_new_sel = X_new_woe[self.selected_features]
            else:
                print("Re-running feature selection...")
                # Re-select features
                features_iv = self.selector.filter_by_iv(self.iv_df, threshold=0.02)
                self.selected_features = self.selector.stepwise_selection(
                    X_new_woe[features_iv], y_new, method='forward'
                )
                X_new_sel = X_new_woe[self.selected_features]
            
            # Incremental learning using partial_fit
            print("Updating model coefficients using partial_fit...")
            self.trainer.partial_fit(X_new_sel, y_new, classes=np.array([0, 1]))
            
            # Re-evaluate model
            print("Re-evaluating model...")
            pred_new = self.trainer.predict_proba(X_new_sel)
            metrics_new = self.evaluator.evaluate(y_new, pred_new)
            
            # Update performance metrics
            self.model_metrics['online'] = metrics_new
            
            # Regenerate scorecard with updated coefficients
            print("Regenerating scorecard...")
            self.scorecard_df = self.sc_transformer.fit(self.trainer.model, self.binning, self.selected_features)
            
        else:
            print("Re-running full pipeline with new data...")
            # Re-run full pipeline (this is slower but more comprehensive)
            # This requires access to original data, which we don't have
            # So we'll just update the model with new data
            print("Warning: Full pipeline re-run requires original data.")
            print("Falling back to incremental learning with new binning...")
            
            # Fit new binning on new data
            new_binning = BinningProcess(
                max_bins=self.binning.max_bins,
                min_bin_pct=self.binning.min_bin_pct,
                special_values=self.binning.special_values
            )
            new_binning.fit(X_new, y_new)
            
            # Transform and select features
            X_new_woe = new_binning.transform(X_new)
            features_iv = self.selector.filter_by_iv(new_binning.iv_df, threshold=0.02)
            self.selected_features = self.selector.stepwise_selection(
                X_new_woe[features_iv], y_new, method='forward'
            )
            X_new_sel = X_new_woe[self.selected_features]
            
            # Train new model
            print("Training new model...")
            self.trainer.train(X_new_sel, y_new)
            
            # Update binning and scorecard
            self.binning = new_binning
            self.iv_df = new_binning.iv_df
            
            # Re-evaluate
            pred_new = self.trainer.predict_proba(X_new_sel)
            metrics_new = self.evaluator.evaluate(y_new, pred_new)
            self.model_metrics['online'] = metrics_new
            
            # Regenerate scorecard
            print("Regenerating scorecard...")
            self.scorecard_df = self.sc_transformer.fit(self.trainer.model, self.binning, self.selected_features)
        
        print(f"Online learning completed.")
        print(f"New performance: AUC={metrics_new['auc']:.4f}, KS={metrics_new['ks']:.4f}")
        
        return self.model_metrics
          
    def predict_proba(self, X):
        X_woe = self.binning.transform(X)
        return self.trainer.predict_proba(X_woe[self.selected_features])
        
    def predict_score(self, X):
        X_woe = self.binning.transform(X)
        return self.sc_transformer.transform(X_woe, self.selected_features, self.trainer.model)
        
    def create_report(self, filename='scorecard_report.xlsx'):
        """
        Export all results to Excel.
        """
        # 1. 数据情况
        data_situation_list = [
            {'Dataset': '原始总样本数', 'Total': self.dataset_stats['original_total']},
            {'Dataset': '样本筛选后最终总样本数', 'Total': self.dataset_stats['filtered_total']},
            {'Dataset': '训练集', 'Total': self.dataset_stats['train']['total'], 
             'Good': self.dataset_stats['train']['good'], 'Bad': self.dataset_stats['train']['bad'], 
             'Bad Rate': f"{self.dataset_stats['train']['bad_rate']:.2%}"},
            {'Dataset': '测试集', 'Total': self.dataset_stats['test']['total'], 
             'Good': self.dataset_stats['test']['good'], 'Bad': self.dataset_stats['test']['bad'], 
             'Bad Rate': f"{self.dataset_stats['test']['bad_rate']:.2%}"}
        ]
        
        if self.dataset_stats['oot'] is not None:
            data_situation_list.append({
                'Dataset': 'OOT集', 'Total': self.dataset_stats['oot']['total'], 
                 'Good': self.dataset_stats['oot']['good'], 'Bad': self.dataset_stats['oot']['bad'], 
                 'Bad Rate': f"{self.dataset_stats['oot']['bad_rate']:.2%}"})
        
        data_situation_df = pd.DataFrame(data_situation_list)
        
        # 2. Summary - 数据集基本信息（合并IV Analysis）
        summary_df = self.data_summary.copy()
        
        # 合并IV值
        if 'iv' in self.iv_df.columns:
            iv_dict = dict(zip(self.iv_df['variable'], self.iv_df['iv']))
            summary_df['IV'] = summary_df['variable'].map(iv_dict)
        
        # 计算特征性能指标（AUC、KS、PSI）
        feature_performance = []
        for feat in self.selected_features:
            if feat in self.binning_stats['train']['feature'].values:
                feat_stats_train = self.binning_stats['train'][self.binning_stats['train']['feature'] == feat]
                feat_stats_test = self.binning_stats['test'][self.binning_stats['test']['feature'] == feat]
                
                # 计算该特征的区分度（使用分箱的坏率差异）
                bad_rates_train = feat_stats_train['bad_rate'].values
                bad_rates_test = feat_stats_test['bad_rate'].values
                
                # 简单的区分度指标（最大坏率 - 最小坏率）
                discrimination_train = bad_rates_train.max() - bad_rates_train.min()
                discrimination_test = bad_rates_test.max() - bad_rates_test.min()
                
                feature_performance.append({
                    'feature': feat,
                    'train_discrimination': discrimination_train,
                    'test_discrimination': discrimination_test
                })
        
        feature_perf_df = pd.DataFrame(feature_performance)
        
        # 合并特征性能指标到summary
        if len(feature_perf_df) > 0:
            perf_dict = dict(zip(feature_perf_df['feature'], feature_perf_df['train_discrimination']))
            summary_df['Feature_Discrimination'] = summary_df['variable'].map(perf_dict)
        
        # 3. Selected Features - 特征选择过程
        # 原始特征清单
        original_features = self.data_summary['variable'].tolist()
        
        # 特征选择过程详情
        selection_process_df = pd.DataFrame(self.selector.selection_process)
        
        # 最终选择的特征清单
        selected_features_df = pd.DataFrame({
            'Feature': self.selected_features,
            'Type': ['Selected'] * len(self.selected_features)
        })
        
        # 4. Model Performance - 更直观的展示
        metrics_list = [
            {'Dataset': '训练集', 'AUC': f"{self.model_metrics['train']['auc']:.4f}", 
             'KS': f"{self.model_metrics['train']['ks']:.4f}", 'Rating': self._get_rating(self.model_metrics['train']['auc'])},
            {'Dataset': '测试集', 'AUC': f"{self.model_metrics['test']['auc']:.4f}", 
             'KS': f"{self.model_metrics['test']['ks']:.4f}", 'Rating': self._get_rating(self.model_metrics['test']['auc']),
             'PSI_vs_Train': f"{self.model_metrics['psi_train_test']:.4f}", 'PSI_Rating': self._get_psi_rating(self.model_metrics['psi_train_test'])}
        ]
        
        if 'oot' in self.model_metrics:
            metrics_list.append({
                'Dataset': 'OOT集', 'AUC': f"{self.model_metrics['oot']['auc']:.4f}", 
                 'KS': f"{self.model_metrics['oot']['ks']:.4f}", 'Rating': self._get_rating(self.model_metrics['oot']['auc']),
                 'PSI_vs_Train': f"{self.model_metrics['psi_train_oot']:.4f}", 'PSI_Rating': self._get_psi_rating(self.model_metrics['psi_train_oot'])
            })
        
        model_performance_df = pd.DataFrame(metrics_list)
        
        # 5. Model Coefficients
        model = self.trainer.model.best_estimator_ if hasattr(self.trainer.model, 'best_estimator_') else self.trainer.model
        coef_df = pd.DataFrame({
            'Feature': ['Intercept'] + self.selected_features,
            'Coefficient': [model.intercept_[0]] + list(model.coef_[0])
        })
        
        # 6. Scorecard - 各数据集的分箱分布
        scorecard_detailed = []
        
        for dataset_name in ['train', 'test']:
            if dataset_name not in self.binning_stats:
                continue
            
            dataset_stats = self.binning_stats[dataset_name]
            
            for feat in self.selected_features:
                feat_stats = dataset_stats[dataset_stats['feature'] == feat]
                
                for _, row in feat_stats.iterrows():
                    scorecard_detailed.append({
                        'Feature': feat,
                        'Dataset': dataset_name,
                        'Bin': row['bin'],
                        'Bin_Range': row['bin_range'],
                        'Count': row['count'],
                        'Good': row['good'],
                        'Bad': row['bad'],
                        'Bad_Rate': f"{row['bad_rate']:.2%}"
                    })
        
        if 'oot' in self.binning_stats:
            dataset_stats = self.binning_stats['oot']
            
            for feat in self.selected_features:
                feat_stats = dataset_stats[dataset_stats['feature'] == feat]
                
                for _, row in feat_stats.iterrows():
                    scorecard_detailed.append({
                        'Feature': feat,
                        'Dataset': 'oot',
                        'Bin': row['bin'],
                        'Bin_Range': row['bin_range'],
                        'Count': row['count'],
                        'Good': row['good'],
                        'Bad': row['bad'],
                        'Bad_Rate': f"{row['bad_rate']:.2%}"
                    })
        
        scorecard_detailed_df = pd.DataFrame(scorecard_detailed)
        
        # 7. Score Bin - 评分等频20分箱分布
        score_bin_detailed = []
        
        for dataset_name in ['train', 'test']:
            if dataset_name not in self.score_stats:
                continue
            
            dataset_stats = self.score_stats[dataset_name]
            
            for _, row in dataset_stats.iterrows():
                score_bin_detailed.append({
                    'Dataset': dataset_name,
                    'Bin': row['bin'],
                    'Score_Range': row['score_range'],
                    'Count': row['count'],
                    'Good': row['good'],
                    'Bad': row['bad'],
                    'Bad_Rate': f"{row['bad_rate']:.2%}",
                    'Cum_Bad_Rate': f"{row['cum_bad_rate']:.2%}",
                    'Cum_Good_Rate': f"{row['cum_good_rate']:.2%}"
                })
        
        if 'oot' in self.score_stats:
            dataset_stats = self.score_stats['oot']
            
            for _, row in dataset_stats.iterrows():
                score_bin_detailed.append({
                    'Dataset': 'oot',
                    'Bin': row['bin'],
                    'Score_Range': row['score_range'],
                    'Count': row['count'],
                    'Good': row['good'],
                    'Bad': row['bad'],
                    'Bad_Rate': f"{row['bad_rate']:.2%}",
                    'Cum_Bad_Rate': f"{row['cum_bad_rate']:.2%}",
                    'Cum_Good_Rate': f"{row['cum_good_rate']:.2%}"
                })
        
        score_bin_detailed_df = pd.DataFrame(score_bin_detailed)
        
        # 8. 原始特征清单
        original_features_df = pd.DataFrame({
            'Feature': original_features,
            'Type': ['Original'] * len(original_features)
        })
        
        data_dict = {
            '数据情况': data_situation_df,
            'Summary': summary_df,
            '原始特征清单': original_features_df,
            '特征选择过程': selection_process_df,
            'Selected Features': selected_features_df,
            'Model Coefficients': coef_df,
            'Model Performance': model_performance_df,
            'Scorecard': scorecard_detailed_df,
            'Score Bin': score_bin_detailed_df
        }
        
        return self.reporter.generate(filename, data_dict)
    
    def _get_rating(self, auc):
        """
        Get rating based on AUC value.
        """
        if auc >= 0.8:
            return '优秀'
        elif auc >= 0.75:
            return '良好'
        elif auc >= 0.7:
            return '中等'
        elif auc >= 0.65:
            return '一般'
        else:
            return '较差'
    
    def _get_psi_rating(self, psi):
        """
        Get PSI rating based on PSI value.
        """
        if psi < 0.1:
            return '稳定'
        elif psi < 0.2:
            return '轻微变化'
        elif psi < 0.25:
            return '中等变化'
        else:
            return '显著变化'
