"""
NanPreprocessing module for handling missing values in datasets.

Imports logging functionality from myffe.tools.Loggerv1_1 import log_class

__init__ receives important parameter shortcut_nan (missing value processing parameters)
__call__ receives important parameter data (dataset for missing value processing)

Uses shortcut_nan with:
- nan_missing_process_choice (missing value processing method)
  (options: fill_nan_mean, fill_nan_median, fill_nan_mode, fill_nan_forward, fill_nan_backward, fill_nan_intelligent, drop_col_high_missing, fill_nan_knn, skip)
- drop_threshold (missing value processing threshold) (range 0-1, default 0.75)
- knn_n_neighbors (KNN neighbors for missing value processing) (default 5)
"""
import sys
import os

# Add project root directory to Python path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')))

from myffe.tools.Loggerv1_1 import log_class
from myffe.tools.color_printer import ColorPrinter
from myffe.tools.evaluation_decorator import with_evaluation

@log_class(type='use')
class NanPreprocessing:
    def __init__(self, shortcut_nan=None):            #---------------------Pass missing value processing parameters
        self.shortcut_nan = shortcut_nan

    @with_evaluation
    def __call__(self, data):
        
        import pandas as pd
        if self.shortcut_nan =='skip':
            ColorPrinter.print_error('错误: 缺少必要的参数 shortcut_nan')
            return data
        data_size = data.memory_usage(deep=True).sum() / (1024 * 1024)  # Convert to MB
        ColorPrinter.print_info(f'数据大小: {data_size:.2f} MB')
        ColorPrinter.print_info(f'数据行数: {len(data)} 行')

        ColorPrinter.print_info('使用 pandas 处理')
        self.data = data
        ColorPrinter.print_info('下面进行缺失值处理')
        processed_data = self.nan_preprocessing_switch(data)
        
        # 只返回特征列，不包含标签列
        return processed_data

    def nan_preprocessing_switch(self, data):        
        #这里是主要参数的初始化
        processing_cols = self.shortcut_nan.get('nan_processing_col')
        processing_methods = self.shortcut_nan.get('nan_missing_process_choice')



        #检查参数是否完整
        if not processing_cols or not processing_methods:
            ColorPrinter.print_error('错误: 缺少必要的参数 nan_processing_col 或 nan_missing_process_choice')
            return data



        #这里是展示缺失值处理的参数
        ColorPrinter.print_info(f"处理列数: {len(processing_cols)}")
        ColorPrinter.print_info(f"处理方法数: {len(processing_methods)}")
        ColorPrinter.print_info(f"处理列: {processing_cols[:5]}...")
        ColorPrinter.print_info(f"处理方法: {processing_methods[:5]}...")
        


        #这里是对于每一列的缺失值处理
        for col, way in zip(processing_cols, processing_methods):
            if col in data.columns:
                # 直接将way转换为字符串
                method = str(way)
                
                ColorPrinter.print_info(f"处理列: {col}, 方法: {method}")
                if method == 'skip':         
                    continue
                elif method == 'drop_col_high_missing':
                    #构建临时参数字典
                    temp_params = {
                        'drop_threshold': self.shortcut_nan.get('drop_threshold', 0.75)
                    }
                    data = self.drop_high_missing_single_col(data, col, temp_params)    #---------------------------------这里是高缺失列直接删除

                elif method == 'fill_nan_knn':
                    # -----------------------------------------------------------------------------KNN 填充
                    if data[col].isna().sum() > 0:
                        data = self.fill_nan_knn(data, target_col=col)
                elif method == 'fill_nan_mean':    #---------------------------------使用pandas处理
                    if data[col].isna().sum() > 0:
                        import pandas as pd
                        if pd.api.types.is_numeric_dtype(data[col]):
                            data[col] = data[col].fillna(data[col].mean())
                            ColorPrinter.print_info(f'fill nan mean: {col}')
                elif method == 'fill_nan_median':    #---------------------------------使用pandas处理
                    if data[col].isna().sum() > 0:
                        import pandas as pd
                        if pd.api.types.is_numeric_dtype(data[col]):
                            data[col] = data[col].fillna(data[col].median())
                            ColorPrinter.print_info(f'fill nan median: {col}')
                elif method == 'fill_nan_mode':    #---------------------------------使用pandas处理
                    if data[col].isna().sum() > 0:
                        mode_value = data[col].mode().iloc[0] if not data[col].mode().empty else None
                        if mode_value is not None:
                            data[col] = data[col].fillna(mode_value)
                            ColorPrinter.print_info(f'fill nan mode: {col} = {mode_value}')
                elif method == 'fill_nan_forward':    #---------------------------------使用pandas处理
                    if data[col].isna().sum() > 0:
                        data[col] = data[col].ffill()
                        ColorPrinter.print_info(f'fill nan forward: {col}')
                elif method == 'fill_nan_backward':    #---------------------------------使用pandas处理
                    if data[col].isna().sum() > 0:
                        data[col] = data[col].bfill()
                        ColorPrinter.print_info(f'fill nan backward: {col}')
                elif method == 'fill_nan_intelligent':    #---------------------------------智能填充
                    if data[col].isna().sum() > 0:
                        data = self.fill_nan_intelligent(data, target_col=col)
                        ColorPrinter.print_info(f'fill nan intelligent: {col}')
                elif method == 'fill_nan_ei':    #---------------------------------集成填充
                    if data[col].isna().sum() > 0:
                        # 获取当前列在处理列表中的索引
                        try:
                            col_index = self.shortcut_nan.get('nan_processing_col', []).index(col)
                            # 获取该列对应的模型类型
                            model_types = self.shortcut_nan.get('ei_model_types', [])
                            model_type = model_types[col_index] if col_index < len(model_types) else 'linear'
                            # 获取该列对应的多项式阶数
                            poly_degrees = self.shortcut_nan.get('poly_degrees', [])
                            poly_degree = poly_degrees[col_index] if col_index < len(poly_degrees) else 2
                        except (ValueError, IndexError):
                            # 如果索引不存在，使用默认值
                            model_type = 'linear'
                            poly_degree = 2
                        data = self.fill_nan_ei(data, target_col=col, poly_degree=poly_degree, model_type=model_type)
                        ColorPrinter.print_info(f'fill nan ei: {col} (model: {model_type}, degree: {poly_degree})')
                elif method == 'specify_fill':    #---------------------------------指定值填充
                    if data[col].isna().sum() > 0:
                        # 获取填充值
                        fill_values = self.shortcut_nan.get('fill_values', [])
                        # 找到当前列在处理列表中的索引
                        try:
                            col_index = self.shortcut_nan.get('nan_processing_col', []).index(col)
                            if col_index < len(fill_values):
                                fill_value = fill_values[col_index]
                                data[col] = data[col].fillna(fill_value)
                                ColorPrinter.print_info(f'fill nan with specified value: {col} = {fill_value}')
                            else:
                                ColorPrinter.print_error(f'列 {col} 没有指定填充值')
                        except ValueError:
                            ColorPrinter.print_error(f'列 {col} 不在处理列表中')
        return data
    

    
    def drop_high_missing_single_col(self, data, col, params):
        """Delete single high missing value column
        
        Args:
            data: pandas DataFrame, input data
            col: str, column name
            params: dict, column parameters
            
        Returns:
            pandas DataFrame, processed data
        """
        threshold = params.get('drop_threshold', 0.75)
        if data[col].isna().sum() / len(data) > threshold:
            data.drop(col, axis=1, inplace=True)
            ColorPrinter.print_info(f'drop column: {col}')
        
        return data



    def fill_nan_knn(self, data, target_col):
        """Fill missing values using KNN imputation
        
        Args:
            data: pandas DataFrame, input data
            target_col: str, column name to fill
            
        Returns:
            pandas DataFrame, processed data
        """
        # Use pandas processing
        import pandas as pd
        
        from sklearn.impute import KNNImputer
        
        feature_cols = [col for col in data.columns] # All columns as feature columns
        numeric_cols = [col for col in feature_cols if pd.api.types.is_numeric_dtype(data[col])] # Here we filter out numeric columns
        
        if not numeric_cols or target_col not in numeric_cols:
            ColorPrinter.print_info(f'Column {target_col} is not a numeric column or does not exist')
            return data
        
        # Create KNNImputer instance
        n_neighbors = self.shortcut_nan.get('knn_n_neighbors', 5)                   # Here set default value to 5, number of neighbors for KNN filling
        imputer = KNNImputer(n_neighbors=n_neighbors)
        
        # KNN filling for all numeric columns
        imputed_data = imputer.fit_transform(data[numeric_cols])
        # Only update target column
        data[target_col] = imputed_data[:, numeric_cols.index(target_col)]
        ColorPrinter.print_info(f'Filling missing values with KNN, number of neighbors: {n_neighbors}')
        ColorPrinter.print_info('Filled columns:', [target_col])
        
        return data

    def fill_nan_intelligent(self, data, target_col):
        """Fill missing values using intelligent imputation based on correlation
        
        Args:
            data: pandas DataFrame, input data
            target_col: str, column name to fill
            
        Returns:
            pandas DataFrame, processed data
        """
        # Use pure Python to implement intelligent filling
        import pandas as pd
        import numpy as np
        
        feature_cols = [col for col in data.columns]
        numeric_cols = [col for col in feature_cols if pd.api.types.is_numeric_dtype(data[col])]
        
        if not numeric_cols or target_col not in numeric_cols:
            ColorPrinter.print_info(f'Column {target_col} is not a numeric column or does not exist')
            return data
        
        # Calculate correlation between features
        corr_matrix = data[numeric_cols].corr() # Here calculate correlation matrix between numeric columns
        
        if data[target_col].isna().sum() > 0: # Here check if current column has missing values
            # Find the column with highest correlation to current column
            corr_with_col = corr_matrix[target_col].sort_values(ascending=False) # Here sort by correlation in descending order
            # Exclude itself
            corr_with_col = corr_with_col[corr_with_col.index != target_col] # Here it looks like idx but for columns it's column names
            
            if len(corr_with_col) > 0:  
                # Select the column with highest correlation
                most_correlated_col = corr_with_col.index[0] # Here get the column name with highest correlation
                correlation = corr_with_col.iloc[0] # Here get the correlation value of the column with highest correlation
                
                # Check correlation threshold
                high_correlation = abs(correlation) > 0.5
                
                if high_correlation:  # Correlation threshold
                    ColorPrinter.print_info(f'Using column {most_correlated_col} with highest correlation to {target_col} (correlation: {correlation:.2f}) for intelligent filling')
                    
                    # Pure Python implementation of intelligent filling
                    ColorPrinter.print_info('Using pure Python for intelligent filling')
                    filled_col = data[target_col].copy()
                    filled_count = 0
                    
                    # Find indices with values
                    valid_indices = ~data[most_correlated_col].isna() & ~data[target_col].isna()
                    if valid_indices.sum() > 0:
                        # Simple linear regression
                        from sklearn.linear_model import LinearRegression
                        X = data.loc[valid_indices, most_correlated_col].values.reshape(-1, 1)
                        y = data.loc[valid_indices, target_col].values
                        
                        model = LinearRegression()
                        model.fit(X, y)
                        
                        # Predict missing values
                        missing_indices = data[target_col].isna()
                        if missing_indices.sum() > 0:
                            X_missing = data.loc[missing_indices, most_correlated_col].values.reshape(-1, 1)
                            predictions = model.predict(X_missing)
                            filled_col.loc[missing_indices] = predictions
                            filled_count = len(predictions)
                    
                    if filled_count > 0:
                        data[target_col] = filled_col
                        ColorPrinter.print_info(f'Intelligently filled column {target_col}, filled {filled_count} missing values')
                    else:
                        ColorPrinter.print_info(f'Column {most_correlated_col} also has missing values, cannot perform intelligent filling')
                else:
                    ColorPrinter.print_info(f'No highly correlated columns with {target_col}, using mean filling')
                    data[target_col] = data[target_col].fillna(data[target_col].mean())
            else:
                ColorPrinter.print_info(f'No other numeric columns, using mean filling')
                data[target_col] = data[target_col].fillna(data[target_col].mean())
        
        return data

    def fill_nan_ei(self, data, target_col, poly_degree=2, model_type='linear'):
        """Fill missing values using Ensemble Imputation (EI) with validation-based weighting
        
        Args:
            data: pandas DataFrame, input data
            target_col: str, column name to fill
            poly_degree: int, degree of polynomial features
            model_type: str, type of model to use
            
        Returns:
            pandas DataFrame, processed data
        """
        # Ensemble Imputation with validation-based weighting and iteration optimization
        import pandas as pd
        import numpy as np
        import random
        import importlib
        from sklearn.preprocessing import OneHotEncoder, LabelEncoder
        from sklearn.preprocessing import PolynomialFeatures
        from sklearn.linear_model import LinearRegression, Ridge, Lasso
        from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
        from sklearn.model_selection import train_test_split
        from sklearn.metrics import mean_squared_error, r2_score
        
        feature_cols = [col for col in data.columns if col != target_col]
        
        # Separate training and prediction data
        train_mask = ~data[target_col].isna()
        test_mask = data[target_col].isna()
        
        # Define different imputation methods
        imputation_methods = ['mean', 'median', 'mode']
        # Randomly select initial iteration count between 20-100
        max_iterations = random.randint(20, 100)
        current_iteration = 0
        
        best_score = -float('inf')
        best_predictions = None
        best_weights = None
        
        # Model dictionary
        model_dict = {
            'linear': LinearRegression(),
            'ridge': Ridge(),
            'lasso': Lasso(),
            'random_forest': RandomForestRegressor(),
            'gradient_boosting': GradientBoostingRegressor()
        }
        
        # Get specified model
        if model_type in model_dict:
            model = model_dict.get(model_type, LinearRegression())
        else:
            # Handle custom model
            try:
                # Parse model path, e.g., sklearn.ensemble.RandomForestRegressor
                parts = model_type.split('.')
                module_name = '.'.join(parts[:-1])
                class_name = parts[-1]
                
                # Dynamically import module
                module = importlib.import_module(module_name)
                # Get model class
                model_class = getattr(module, class_name)
                # Instantiate model
                model = model_class()
            except Exception as e:
                # If import fails, use default model
                ColorPrinter.print_error(f'Custom model import failed: {e}, using default model LinearRegression')
                model = LinearRegression()
        
        while current_iteration < max_iterations:
            all_predictions = []
            method_weights = []
            all_val_scores = []
            
            for method in imputation_methods:
                # Create data copy
                data_copy = data.copy()
                
                # Temporarily fill non-target columns
                for col in feature_cols:
                    if data_copy[col].isna().sum() > 0:
                        if pd.api.types.is_numeric_dtype(data_copy[col]):
                            if method == 'mean':
                                data_copy[col] = data_copy[col].fillna(data_copy[col].mean())
                            elif method == 'median':
                                data_copy[col] = data_copy[col].fillna(data_copy[col].median())
                            elif method == 'mode':
                                mode_value = data_copy[col].mode().iloc[0] if not data_copy[col].mode().empty else 0
                                data_copy[col] = data_copy[col].fillna(mode_value)
                        else:
                            # Categorical columns use mode filling
                            mode_value = data_copy[col].mode().iloc[0] if not data_copy[col].mode().empty else 'unknown'
                            data_copy[col] = data_copy[col].fillna(mode_value)
                
                # Feature encoding
                encoded_features = []
                for col in feature_cols:
                    # Check if it's a categorical, object, or string column
                    if pd.api.types.is_categorical_dtype(data_copy[col]) or pd.api.types.is_object_dtype(data_copy[col]) or str(data_copy[col].dtype) == 'string' or str(data_copy[col].dtype) == 'str':
                        # Categorical column encoding
                        le = LabelEncoder()
                        try:
                            # Ensure all values are strings
                            str_col = data_copy[col].astype(str)
                            # Handle possible missing values
                            str_col = str_col.fillna('unknown')
                            encoded_col = le.fit_transform(str_col)
                            # Ensure it's a numpy array before reshaping
                            encoded_col = np.array(encoded_col)
                            encoded_features.append(encoded_col.reshape(-1, 1))
                        except Exception as e:
                            pass
                    # Check if it's a datetime column
                    elif pd.api.types.is_datetime64_any_dtype(data_copy[col]):
                        # Datetime column encoding
                        try:
                            encoded_features.append(np.array(data_copy[col].dt.year).reshape(-1, 1))
                            encoded_features.append(np.array(data_copy[col].dt.month).reshape(-1, 1))
                            encoded_features.append(np.array(data_copy[col].dt.day).reshape(-1, 1))
                            if hasattr(data_copy[col].dt, 'hour'):
                                encoded_features.append(np.array(data_copy[col].dt.hour).reshape(-1, 1))
                            else:
                                encoded_features.append(np.zeros((len(data_copy), 1)))
                        except Exception as e:
                            pass
                    # Otherwise consider it a numeric column
                    else:
                        try:
                            # Ensure it's a numpy array before reshaping
                            col_values = np.array(data_copy[col], dtype=np.float64)
                            # Handle possible NaN values
                            col_values = np.nan_to_num(col_values, nan=0.0)
                            encoded_features.append(col_values.reshape(-1, 1))
                        except Exception as e:
                            pass
                
                if not encoded_features:
                    continue
                
                # Combine features
                X = np.hstack(encoded_features)
                y = data_copy[target_col].values
                
                # Polynomial features
                poly = PolynomialFeatures(degree=poly_degree, include_bias=False)
                X_poly = poly.fit_transform(X)
                
                # Split into training and validation sets
                X_train, X_val, y_train, y_val = train_test_split(
                    X_poly[train_mask], y[train_mask], test_size=0.2, random_state=42
                )
                
                # Train model
                model.fit(X_train, y_train)
                
                # Evaluate model performance on validation set
                val_predictions = model.predict(X_val)
                mse = mean_squared_error(y_val, val_predictions)
                r2 = r2_score(y_val, val_predictions)
                all_val_scores.append(r2)
                
                # Calculate weights (smaller MSE means higher weight)
                weight = 1.0 / (mse + 1e-10)  # Avoid division by zero
                method_weights.append(weight)
                
                # Predict
                if test_mask.sum() > 0:
                    predictions = model.predict(X_poly[test_mask])
                    all_predictions.append(predictions)
            
            # Check if there are valid predictions
            if all_predictions:
                # Normalize weights
                total_weight = sum(method_weights)
                normalized_weights = [w / total_weight for w in method_weights]
                
                # Calculate weighted average
                weighted_predictions = np.zeros_like(all_predictions[0])
                for pred, weight in zip(all_predictions, normalized_weights):
                    weighted_predictions += pred * weight
                
                # Calculate average validation score
                avg_val_score = np.mean(all_val_scores)
                
                # Calculate prediction similarity
                predictions_array = np.array(all_predictions)
                similarity = 100 - np.std(predictions_array, axis=0).mean() * 100
                if similarity < 0:
                    similarity = 0
                
                # Update best results based on validation score
                if avg_val_score > best_score:
                    best_score = avg_val_score
                    best_predictions = weighted_predictions
                    best_weights = normalized_weights
                
                # Adjust iteration count
                if avg_val_score < 0.6:
                    max_iterations += 5
                    ColorPrinter.print_info(f'Validation score is low, increasing iteration count to {max_iterations}')
                elif avg_val_score > 0.85:
                    max_iterations -= 5
                    ColorPrinter.print_info(f'Validation score is high, decreasing iteration count to {max_iterations}')
                
                # Check iteration count range
                if max_iterations < 5 or max_iterations > 1000:
                    ColorPrinter.print_info(f'Iteration count out of range ({max_iterations}), stopping iteration')
                    # Use current best calculation result
                    ColorPrinter.print_info('Using current best calculation result')
                    break
            
            current_iteration += 1
        
        # Use best prediction results
        if best_predictions is not None:
            data.loc[test_mask, target_col] = best_predictions
            ColorPrinter.print_info(f'EI filled column {target_col}, filled {len(best_predictions)} missing values')
            ColorPrinter.print_info(f'Filling method weights: mean={best_weights[0]:.4f}, median={best_weights[1]:.4f}, mode={best_weights[2]:.4f}')
        else:
            # Fallback to mean filling
            if pd.api.types.is_numeric_dtype(data[target_col]):
                data[target_col] = data[target_col].fillna(data[target_col].mean())
            else:
                mode_value = data[target_col].mode().iloc[0] if not data[target_col].mode().empty else None
                if mode_value is not None:
                    data[target_col] = data[target_col].fillna(mode_value)
            ColorPrinter.print_info(f'EI filling failed, falling back to default filling method')
        
        return data


"""
Parameter documentation:

shortcut_nan parameter dictionary:
    - nan_missing_process_choice: Missing value processing method choice
      Type: list of lists
      Optional values: 'fill_nan_mean', 'fill_nan_median', 'fill_nan_mode', 'fill_nan_forward', 'fill_nan_backward', 'fill_nan_intelligent', 'drop_nan', 'drop_col_high_missing', 'fill_nan_knn', 'skip'
      Default: None (must be specified)
      Description: Specify processing method for each column, list length must be the same as nan_processing_col
        - 'fill_nan_mean': Fill missing values with mean
        - 'fill_nan_median': Fill missing values with median
        - 'fill_nan_mode': Fill missing values with mode
        - 'fill_nan_forward': Fill missing values with forward fill
        - 'fill_nan_backward': Fill missing values with backward fill
        - 'fill_nan_intelligent': Fill missing values with intelligent imputation
        - 'drop_col_high_missing': Drop columns with high missing value ratio
        - 'fill_nan_knn': Fill missing values with KNN algorithm
        - 'fill_nan_ei': Fill missing values with ensemble imputation algorithm
        - 'specify_fill': Fill missing values with user-specified values
        - 'skip': Skip missing value processing for this column
    
    - nan_processing_col: Columns to process
      Type: list
      Optional values: Column names in the dataset
      Default: None (must be specified)
      Description: Specify columns to perform missing value processing on, list length must be the same as nan_missing_process_choice
    
    - drop_threshold: Missing value ratio threshold
      Type: float
      Optional values: 0.0 - 1.0
      Default: 0.5
      Description: When the column's missing value ratio exceeds this threshold, drop the column (only used in drop_col_high_missing mode)
    
    - knn_n_neighbors: Number of neighbors for KNN algorithm
      Type: integer
      Optional values: >= 1
      Default: 5
      Description: Number of neighbors used for KNN filling (only used in fill_nan_knn mode)

Usage example:
    # Apply different missing value processing methods to different columns
    nan_params = {
        'nan_missing_process_choice': [
            ['drop_nan', 'fill_nan_knn'],      # Processing method for numeric column 1
            ['drop_nan', 'drop_col_high_missing', 'fill_nan_knn'],  # Processing method for numeric column 2
            ['drop_nan', 'fill_nan_mode'],      # Processing method for categorical column
            ['drop_nan', 'skip']                # Processing method for column without missing values
        ],
        'nan_processing_col': ['numeric_col1', 'numeric_col2', 'categorical_col', 'datetime_col'],
        'drop_threshold': 0.4,
        'knn_n_neighbors': 5
    }
    nan_processor = NanPreprocessing(shortcut_nan=nan_params)
    cleaned_data = nan_processor(data)
"""


if __name__ == "__main__":
    import pandas as pd
    import numpy as np
    
    ColorPrinter.print_any("=" * 60)
    ColorPrinter.print_info("缺失值处理测试程序")
    ColorPrinter.print_any("=" * 60)
    
    # 创建包含缺失值的测试数据
    np.random.seed(42)
    test_data = pd.DataFrame({
        'feature1': [1, 2, np.nan, 4, 5, np.nan, 7, 8, 9, 10],
        'feature2': [np.nan, 12, 13, np.nan, 15, 16, np.nan, 18, 19, 20],
        'feature3': [21, 22, 23, 24, 25, 26, 27, 28, np.nan, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    ColorPrinter.print_any("\n原始数据:")
    ColorPrinter.print_any(test_data)
    ColorPrinter.print_any("\n缺失值统计:")
    ColorPrinter.print_any(test_data.isnull().sum())
    
    # 测试不同的缺失值处理方式
    test_cases = [
        {
            'nan_missing_process_choice': 'fill_nan_mean',
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'fill_nan_median',
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'drop_nan',
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'drop_col_high_missing',
            'drop_threshold': 0.3
        },
        {
            'nan_missing_process_choice': ['fill_nan_mean', 'drop_col_high_missing'],
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'fill_nan_knn',
            'drop_threshold': 0.5,
            'knn_n_neighbors': 3
        }
    ]
    
    for i, params in enumerate(test_cases, 1):
        ColorPrinter.print_any(f"\n{'=' * 60}")
        ColorPrinter.print_info(f"测试 {i}: {params['nan_missing_process_choice']}")
        ColorPrinter.print_any('=' * 60)
        
        nan_processor = NanPreprocessing(shortcut_nan=params)
        cleaned_data = nan_processor(test_data.copy(), 'normal', 'label')
        
        ColorPrinter.print_any("\n处理后的数据:")
        ColorPrinter.print_any(cleaned_data)
        ColorPrinter.print_any("\n处理后缺失值统计:")
        ColorPrinter.print_any(cleaned_data.isnull().sum())
    
    # 全面渗透测试套件
    ColorPrinter.print_any(f"\n{'=' * 80}")
    ColorPrinter.print_info("全面渗透测试套件")
    ColorPrinter.print_any('=' * 80)
    
    # 测试1: 列级参数测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试1: 列级参数测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_col_level = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_col_level = {
        'nan_missing_process_choice': [['fill_nan_mean'], ['fill_nan_median'], ['fill_nan_mode']],
        'nan_processing_col': ['col1', 'col2', 'col3'],
        'drop_threshold': 0.5
    }
    
    nan_processor_col_level = NanPreprocessing(shortcut_nan=params_col_level)
    result_col_level = nan_processor_col_level(test_data_col_level.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_col_level)
    ColorPrinter.print_any(f"\n数据形状变化: {test_data_col_level.shape} -> {result_col_level.shape}")
    ColorPrinter.print_info("期望：col1均值填充，col2中位数填充，col3众数填充")
    
    # 测试2: 前向和后向填充测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试2: 前向和后向填充测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_forward_backward = pd.DataFrame({
        'col1': [1, np.nan, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, 14, np.nan, np.nan, 17, 18, 19, 20],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_forward_backward = {
        'nan_missing_process_choice': [['fill_nan_forward'], ['fill_nan_backward']],
        'nan_processing_col': ['col1', 'col2'],
        'drop_threshold': 0.5
    }
    
    nan_processor_forward_backward = NanPreprocessing(shortcut_nan=params_forward_backward)
    result_forward_backward = nan_processor_forward_backward(test_data_forward_backward.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_forward_backward)
    ColorPrinter.print_info("期望：col1前向填充，col2后向填充")
    
    # 测试3: 智能填充测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试3: 智能填充测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_intelligent = pd.DataFrame({
        'col1': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
        'col2': [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.1],
        'col3': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_intelligent = {
        'nan_missing_process_choice': 'fill_nan_intelligent',
        'drop_threshold': 0.5
    }
    
    nan_processor_intelligent = NanPreprocessing(shortcut_nan=params_intelligent)
    result_intelligent = nan_processor_intelligent(test_data_intelligent.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_intelligent)
    ColorPrinter.print_info("期望：col3使用col1或col2进行智能填充")
    
    # 测试4: 删除高缺失值列测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试4: 删除高缺失值列测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_drop_col = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, np.nan, 14, 15, np.nan, 17, 18, np.nan, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_drop_col = {
        'nan_missing_process_choice': 'drop_col_high_missing',
        'drop_threshold': 0.3
    }
    
    nan_processor_drop_col = NanPreprocessing(shortcut_nan=params_drop_col)
    result_drop_col = nan_processor_drop_col(test_data_drop_col.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_drop_col)
    ColorPrinter.print_any(f"\n数据形状变化: {test_data_drop_col.shape} -> {result_drop_col.shape}")
    ColorPrinter.print_info("期望：删除col2（缺失值比例33.3% > 30%阈值）")
    
    # 测试5: KNN填充测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试5: KNN填充测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_knn = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_knn = {
        'nan_missing_process_choice': 'fill_nan_knn',
        'drop_threshold': 0.5,
        'knn_n_neighbors': 3
    }
    
    nan_processor_knn = NanPreprocessing(shortcut_nan=params_knn)
    result_knn = nan_processor_knn(test_data_knn.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_knn)
    ColorPrinter.print_info("期望：col1和col2使用KNN填充缺失值")
    
    # 测试6: 组合处理测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试6: 组合处理测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_combined = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_combined = {
        'nan_missing_process_choice': [['fill_nan_mean', 'drop_col_high_missing'], ['fill_nan_median', 'skip']],
        'nan_processing_col': ['col1', 'col2', 'col3'],
        'drop_threshold': 0.3
    }
    
    nan_processor_combined = NanPreprocessing(shortcut_nan=params_combined)
    result_combined = nan_processor_combined(test_data_combined.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_combined)
    ColorPrinter.print_info("期望：col1均值填充，col2中位数填充，col3删除（如果缺失值比例高）")
    
    # 测试7: 边界情况测试 - 无缺失值
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试7: 边界情况测试 - 无缺失值")
    ColorPrinter.print_any('-' * 60)
    
    test_data_no_missing = pd.DataFrame({
        'col1': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_no_missing = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_no_missing = NanPreprocessing(shortcut_nan=params_no_missing)
    result_no_missing = nan_processor_no_missing(test_data_no_missing.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_no_missing)
    ColorPrinter.print_info("期望：数据保持不变，无缺失值处理")
    
    # 测试8: 边界情况测试 - 全部缺失值
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试8: 边界情况测试 - 全部缺失值")
    ColorPrinter.print_any('-' * 60)
    
    test_data_all_missing = pd.DataFrame({
        'col1': [np.nan, np.nan, np.nan, np.nan, np.nan],
        'col2': [np.nan, np.nan, np.nan, np.nan, np.nan],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    params_all_missing = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_all_missing = NanPreprocessing(shortcut_nan=params_all_missing)
    result_all_missing = nan_processor_all_missing(test_data_all_missing.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_all_missing)
    ColorPrinter.print_info("期望：均值填充，但可能无法计算均值")
    
    # 测试9: 异常情况测试 - 空数据
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试9: 异常情况测试 - 空数据")
    ColorPrinter.print_any('-' * 60)
    
    test_data_empty = pd.DataFrame({
        'col1': [],
        'col2': [],
        'label': []
    })
    
    params_empty = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    try:
        nan_processor_empty = NanPreprocessing(shortcut_nan=params_empty)
        result_empty = nan_processor_empty(test_data_empty.copy(), 'normal', 'label')
        
        ColorPrinter.print_any("\n处理结果:")
        ColorPrinter.print_any(result_empty)
        ColorPrinter.print_info("期望：空数据保持不变")
    except Exception as e:
        ColorPrinter.print_error(f"异常处理: {e}")
        ColorPrinter.print_info("期望：空数据应该能正常处理")
    
    # 测试10: 不同数据类型测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试10: 不同数据类型测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_types = pd.DataFrame({
        'numeric_col': [1, 2, np.nan, 4, 5],
        'string_col': ['A', 'B', np.nan, 'D', 'E'],
        'float_col': [1.1, 2.2, np.nan, 4.4, 5.5],
        'int_col': [10, 20, np.nan, 40, 50],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    params_types = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_types = NanPreprocessing(shortcut_nan=params_types)
    result_types = nan_processor_types(test_data_types.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_types)
    ColorPrinter.print_info("期望：数值列均值填充，字符串列跳过或特殊处理")
    
    # 测试11: 大数据集测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试11: 大数据集测试")
    ColorPrinter.print_any('-' * 60)
    
    np.random.seed(42)
    large_data_size = 10000
    test_data_large = pd.DataFrame({
        'feature1': np.random.randn(large_data_size),
        'feature2': np.random.randn(large_data_size),
        'feature3': np.random.randn(large_data_size),
        'label': np.random.choice(['A', 'B'], large_data_size)
    })
    
    # 随机添加缺失值
    missing_indices1 = np.random.choice(large_data_size, int(large_data_size * 0.1), replace=False)
    missing_indices2 = np.random.choice(large_data_size, int(large_data_size * 0.15), replace=False)
    test_data_large.loc[missing_indices1, 'feature1'] = np.nan
    test_data_large.loc[missing_indices2, 'feature2'] = np.nan
    
    params_large = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_large = NanPreprocessing(shortcut_nan=params_large)
    result_large = nan_processor_large(test_data_large.copy(), 'normal', 'label')
    
    ColorPrinter.print_any(f"\n处理结果（数据大小: {large_data_size} 行）:")
    ColorPrinter.print_any(f"原始缺失值: {test_data_large.isnull().sum().sum()}")
    ColorPrinter.print_any(f"处理后缺失值: {result_large.isnull().sum().sum()}")
    ColorPrinter.print_info("期望：所有缺失值被填充")
    
    # 测试12: 极端阈值测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试12: 极端阈值测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_threshold = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, np.nan, 14, 15, np.nan, 17, 18, np.nan, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_threshold_low = {
        'nan_missing_process_choice': 'drop_col_high_missing',
        'drop_threshold': 0.1  # 极低阈值
    }
    
    params_threshold_high = {
        'nan_missing_process_choice': 'drop_col_high_missing',
        'drop_threshold': 0.9  # 极高阈值
    }
    
    nan_processor_threshold_low = NanPreprocessing(shortcut_nan=params_threshold_low)
    result_threshold_low = nan_processor_threshold_low(test_data_threshold.copy(), 'normal', 'label')
    
    nan_processor_threshold_high = NanPreprocessing(shortcut_nan=params_threshold_high)
    result_threshold_high = nan_processor_threshold_high(test_data_threshold.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n低阈值（0.1）结果:")
    ColorPrinter.print_any(f"删除列数: {test_data_threshold.shape[1] - result_threshold_low.shape[1]}")
    ColorPrinter.print_any("高阈值（0.9）结果:")
    ColorPrinter.print_any(f"删除列数: {test_data_threshold.shape[1] - result_threshold_high.shape[1]}")
    
    # 测试13: 标签列保护测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试13: 标签列保护测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_label = pd.DataFrame({
        'feature1': [1, 2, np.nan, 4, 5],
        'feature2': [11, 12, np.nan, 14, 15],
        'label': [np.nan, 'B', np.nan, 'B', 'A']  # 标签列也有缺失值
    })
    
    params_label = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_label = NanPreprocessing(shortcut_nan=params_label)
    result_label = nan_processor_label(test_data_label.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_label)
    ColorPrinter.print_info("期望：标签列的缺失值不被处理")
    
    # 测试14: 多列同时处理测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试14: 多列同时处理测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_multi = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'col4': [31, 32, 33, 34, 35, 36, 37, 38, 39, 40],
        'col5': [41, 42, 43, 44, 45, 46, 47, 48, 49, 50],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_multi = {
        'nan_missing_process_choice': [
            ['fill_nan_mean'], 
            ['fill_nan_median'], 
            ['fill_nan_mode'], 
            ['fill_nan_forward'], 
            ['fill_nan_backward']
        ],
        'nan_processing_col': ['col1', 'col2', 'col3', 'col4', 'col5'],
        'drop_threshold': 0.5
    }
    
    nan_processor_multi = NanPreprocessing(shortcut_nan=params_multi)
    result_multi = nan_processor_multi(test_data_multi.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(f"数据形状: {test_data_multi.shape} -> {result_multi.shape}")
    ColorPrinter.print_info("期望：每列使用不同的填充方法")
    
    # 测试15: 性能测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试15: 性能测试")
    ColorPrinter.print_any('-' * 60)
    
    import time
    test_data_perf = pd.DataFrame({
        'feature1': np.random.randn(5000),
        'feature2': np.random.randn(5000),
        'feature3': np.random.randn(5000),
        'label': np.random.choice(['A', 'B'], 5000)
    })
    
    # 随机添加缺失值
    for col in ['feature1', 'feature2', 'feature3']:
        missing_indices = np.random.choice(5000, int(5000 * 0.1), replace=False)
        test_data_perf.loc[missing_indices, col] = np.nan
    
    params_perf = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_perf = NanPreprocessing(shortcut_nan=params_perf)
    
    start_time = time.time()
    result_perf = nan_processor_perf(test_data_perf.copy(), 'normal', 'label')
    end_time = time.time()
    
    ColorPrinter.print_any(f"\n处理时间: {end_time - start_time:.4f} 秒")
    ColorPrinter.print_any(f"数据大小: {len(test_data_perf)} 行")
    ColorPrinter.print_any(f"原始缺失值: {test_data_perf.isnull().sum().sum()}")
    ColorPrinter.print_any(f"处理后缺失值: {result_perf.isnull().sum().sum()}")
    
    ColorPrinter.print_any("\n" + "=" * 80)
    ColorPrinter.print_success("全面渗透测试完成！")
    ColorPrinter.print_any("=" * 80)
