"""
导入from MFM.FFE.FFEv3_2.tools.Loggerv1_1 import log_class的日志功能
init 传入重要的参数shortcut_nan(缺失值处理的参数)
call 传入重要的参数data(缺失值处理的数据集), clean_type(缺失值处理的类型), data_label(缺失值处理的标签)
使用shortcut_nan中的
nan_missing_process_choice(缺失值处理的方式)
(有fill_nan_mean, fill_nan_median, fill_nan_mode, fill_nan_forward, fill_nan_backward, fill_nan_intelligent, drop_col_high_missing, fill_nan_knn, skip)
drop_threshold(缺失值处理的阈值)(范围0-1,默认0.75)
knn_n_neighbors(缺失值处理的KNN邻居数)(默认5)   
"""
import sys
import os

# 添加项目根目录到Python路径
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')))

from myffe.tools.Loggerv1_1 import log_class
from myffe.tools.color_printer import ColorPrinter
from myffe.tools.evaluation_decorator import with_evaluation

@log_class(type='use')
class NanPreprocessing:
    def __init__(self, shortcut_nan=None):            #---------------------传入缺失值处理的参数
        self.shortcut_nan = shortcut_nan

    @with_evaluation
    def __call__(self, data, data_label=None):
        
        #--------------------------------------
        import pandas as pd
        self.data_label = data_label
        #-------------------------------------模块导入以及参数的初始化
        # ____________________________________
        if self.shortcut_nan =='skip':
            ColorPrinter.print_error('错误: 缺少必要的参数 shortcut_nan')
            return data
        # ____________________________________判断是否skip跳过



        # -----------------------------------------检查数据大小
        data_size = data.memory_usage(deep=True).sum() / (1024 * 1024)  # 转换为MB
        ColorPrinter.print_info(f'数据大小: {data_size:.2f} MB')
        ColorPrinter.print_info(f'数据行数: {len(data)} 行')
        #-----------------------------------------使用 pandas 处理
        ColorPrinter.print_info('使用 pandas 处理')
        self.data = data
        ColorPrinter.print_info('下面进行缺失值处理')
        return self.nan_preprocessing_switch(self.data)

    def nan_preprocessing_switch(self, data):        
        #-----------------------------------------这里是主要参数的初始化
        processing_cols = self.shortcut_nan.get('nan_processing_col')
        processing_methods = self.shortcut_nan.get('nan_missing_process_choice')



        # -----------------------------------------检查参数是否完整
        if not processing_cols or not processing_methods:
            ColorPrinter.print_error('错误: 缺少必要的参数 nan_processing_col 或 nan_missing_process_choice')
            return data



        #-------------------------------------------这里是展示缺失值处理的参数
        ColorPrinter.print_info(f"处理列数: {len(processing_cols)}")
        ColorPrinter.print_info(f"处理方法数: {len(processing_methods)}")
        ColorPrinter.print_info(f"处理列: {processing_cols[:5]}...")
        ColorPrinter.print_info(f"处理方法: {processing_methods[:5]}...")
        


        #-------------------------------------------这里是对于每一列的缺失值处理
        for col, way in zip(processing_cols, processing_methods):
            if col in data.columns:
                # 直接将way转换为字符串
                method = str(way)
                
                ColorPrinter.print_info(f"处理列: {col}, 方法: {method}")
                if method == 'skip':         
                    continue
                elif method == 'drop_col_high_missing':
                    # -----------------------------------------------------------------------------构建临时参数字典
                    temp_params = {
                        'drop_threshold': self.shortcut_nan.get('drop_threshold', 0.75)
                    }
                    
                    data = self.drop_high_missing_single_col(data, col, temp_params)    #---------------------------------这里是高缺失列直接删除


                elif method == 'fill_nan_knn':
                    # -----------------------------------------------------------------------------KNN 填充
                    if col != self.data_label and data[col].isna().sum() > 0:
                        data = self.fill_nan_knn(data, target_col=col)


                elif method == 'fill_nan_mean':    #---------------------------------使用pandas处理
                    if col != self.data_label and data[col].isna().sum() > 0:
                        import pandas as pd
                        if pd.api.types.is_numeric_dtype(data[col]):
                            data[col] = data[col].fillna(data[col].mean())
                            ColorPrinter.print_info(f'fill nan mean: {col}')
                elif method == 'fill_nan_median':    #---------------------------------使用pandas处理
                    if col != self.data_label and data[col].isna().sum() > 0:
                        import pandas as pd
                        if pd.api.types.is_numeric_dtype(data[col]):
                            data[col] = data[col].fillna(data[col].median())
                            ColorPrinter.print_info(f'fill nan median: {col}')
                elif method == 'fill_nan_mode':    #---------------------------------使用pandas处理
                    if col != self.data_label and data[col].isna().sum() > 0:
                        mode_value = data[col].mode().iloc[0] if not data[col].mode().empty else None
                        if mode_value is not None:
                            data[col] = data[col].fillna(mode_value)
                            ColorPrinter.print_info(f'fill nan mode: {col} = {mode_value}')
                elif method == 'fill_nan_forward':    #---------------------------------使用pandas处理
                    if col != self.data_label and data[col].isna().sum() > 0:
                        data[col] = data[col].ffill()
                        ColorPrinter.print_info(f'fill nan forward: {col}')
                elif method == 'fill_nan_backward':    #---------------------------------使用pandas处理
                    if col != self.data_label and data[col].isna().sum() > 0:
                        data[col] = data[col].bfill()
                        ColorPrinter.print_info(f'fill nan backward: {col}')
                elif method == 'fill_nan_intelligent':    #---------------------------------智能填充
                    if col != self.data_label and data[col].isna().sum() > 0:
                        data = self.fill_nan_intelligent(data, target_col=col)
                        ColorPrinter.print_info(f'fill nan intelligent: {col}')
                elif method == 'fill_nan_ei':    #---------------------------------集成填充
                    if col != self.data_label and data[col].isna().sum() > 0:
                        poly_degree = self.shortcut_nan.get('poly_degree', 2)
                        data = self.fill_nan_ei(data, target_col=col, poly_degree=poly_degree)
                        ColorPrinter.print_info(f'fill nan ei: {col} (degree: {poly_degree})')
                elif method == 'specify_fill':    #---------------------------------指定值填充
                    if col != self.data_label and data[col].isna().sum() > 0:
                        # 获取填充值
                        fill_values = self.shortcut_nan.get('fill_values', {})
                        if col in fill_values:
                            fill_value = fill_values[col]
                            data[col] = data[col].fillna(fill_value)
                            ColorPrinter.print_info(f'fill nan with specified value: {col} = {fill_value}')
                        else:
                            ColorPrinter.print_error(f'列 {col} 没有指定填充值')
        return data
    

    
    def drop_high_missing_single_col(self, data, col, params):
        """删除单个高缺失值列
        
        Args:
            data: pandas DataFrame，输入数据
            col: str，列名
            params: dict，列参数
            
        Returns:
0            pandas DataFrame，处理后的数据
        """
        if col == self.data_label:
            return data
        
        threshold = params.get('drop_threshold', 0.75)
        if data[col].isna().sum() / len(data) > threshold:
            data.drop(col, axis=1, inplace=True)
            ColorPrinter.print_info(f'drop column: {col}')
        
        return data



    def fill_nan_knn(self, data, target_col):
        # 使用pandas处理
        import pandas as pd
        
        from sklearn.impute import KNNImputer
        
        feature_cols = [col for col in data.columns if col != self.data_label] #这里将标签列从特征列中排除
        numeric_cols = [col for col in feature_cols if pd.api.types.is_numeric_dtype(data[col])] #这里筛选出数值列
        
        if not numeric_cols or target_col not in numeric_cols:
            ColorPrinter.print_info(f'列 {target_col} 不是数值列或不存在')
            return data
        
        # 创建KNNImputer实例
        n_neighbors = self.shortcut_nan.get('knn_n_neighbors', 5)                   #这里设置默认值为5，knn填充的邻居数
        imputer = KNNImputer(n_neighbors=n_neighbors)
        
        # 对所有数值列进行KNN填充
        imputed_data = imputer.fit_transform(data[numeric_cols])
        # 只更新目标列
        data[target_col] = imputed_data[:, numeric_cols.index(target_col)]
        ColorPrinter.print_info(f'使用KNN填充缺失值，邻居数: {n_neighbors}')
        ColorPrinter.print_info('填充的列:', [target_col])
        
        return data

    def fill_nan_intelligent(self, data, target_col):
        # 使用纯 Python 实现智能填充
        import pandas as pd
        import numpy as np
        
        feature_cols = [col for col in data.columns if col != self.data_label]
        numeric_cols = [col for col in feature_cols if pd.api.types.is_numeric_dtype(data[col])]
        
        if not numeric_cols or target_col not in numeric_cols:
            ColorPrinter.print_info(f'列 {target_col} 不是数值列或不存在')
            return data
        
        # 计算特征间的相关性
        corr_matrix = data[numeric_cols].corr() #这里计算数值列之间的相关性矩阵
        
        if data[target_col].isna().sum() > 0: #这里判断当前列是否有缺失值
            # 找到与当前列相关性最高的列
            corr_with_col = corr_matrix[target_col].sort_values(ascending=False) #这里按相关性降序排序
            # 排除自身
            corr_with_col = corr_with_col[corr_with_col.index != target_col] #这里看似是idx其实对于列是列名
            
            if len(corr_with_col) > 0:  
                # 选择相关性最高的列
                most_correlated_col = corr_with_col.index[0] #这里获取相关性最高的列名
                correlation = corr_with_col.iloc[0] #这里获取相关性最高的列的相关性值
                
                # 检查相关性阈值
                high_correlation = abs(correlation) > 0.5
                
                if high_correlation:  # 相关性阈值
                    ColorPrinter.print_info(f'使用与 {target_col} 相关性最高的列 {most_correlated_col} (相关性: {correlation:.2f}) 进行智能填充')
                    
                    # 纯 Python 实现智能填充
                    ColorPrinter.print_info('使用纯 Python 实现智能填充')
                    filled_col = data[target_col].copy()
                    filled_count = 0
                    
                    # 找到有值的索引
                    valid_indices = ~data[most_correlated_col].isna() & ~data[target_col].isna()
                    if valid_indices.sum() > 0:
                        # 简单线性回归
                        from sklearn.linear_model import LinearRegression
                        X = data.loc[valid_indices, most_correlated_col].values.reshape(-1, 1)
                        y = data.loc[valid_indices, target_col].values
                        
                        model = LinearRegression()
                        model.fit(X, y)
                        
                        # 预测缺失值
                        missing_indices = data[target_col].isna()
                        if missing_indices.sum() > 0:
                            X_missing = data.loc[missing_indices, most_correlated_col].values.reshape(-1, 1)
                            predictions = model.predict(X_missing)
                            filled_col.loc[missing_indices] = predictions
                            filled_count = len(predictions)
                    
                    if filled_count > 0:
                        data[target_col] = filled_col
                        ColorPrinter.print_info(f'智能填充列 {target_col}，填充了 {filled_count} 个缺失值')
                    else:
                        ColorPrinter.print_info(f'列 {most_correlated_col} 也有缺失值，无法进行智能填充')
                else:
                    ColorPrinter.print_info(f'没有与 {target_col} 高度相关的列，使用均值填充')
                    data[target_col] = data[target_col].fillna(data[target_col].mean())
            else:
                ColorPrinter.print_info(f'没有其他数值列，使用均值填充')
                data[target_col] = data[target_col].fillna(data[target_col].mean())
        
        return data

    def fill_nan_ei(self, data, target_col, poly_degree=2):
        # 集成填充 (Ensemble Imputation) with validation-based weighting and iteration optimization
        import pandas as pd
        import numpy as np
        import random
        from sklearn.preprocessing import OneHotEncoder, LabelEncoder
        from sklearn.preprocessing import PolynomialFeatures
        from sklearn.linear_model import LinearRegression
        from sklearn.model_selection import train_test_split
        from sklearn.metrics import mean_squared_error, r2_score
        
        feature_cols = [col for col in data.columns if col != target_col]
        
        # 分离训练数据和预测数据
        train_mask = ~data[target_col].isna()
        test_mask = data[target_col].isna()
        
        # 定义不同的填充方法
        imputation_methods = ['mean', 'median', 'mode']
        # 从20-100之间随机选择初始迭代次数
        max_iterations = random.randint(20, 100)
        current_iteration = 0
        
        best_score = -float('inf')
        best_predictions = None
        best_weights = None
        
        while current_iteration < max_iterations:
            all_predictions = []
            method_weights = []
            all_val_scores = []
            
            for method in imputation_methods:
                # 创建数据副本
                data_copy = data.copy()
                
                # 对非目标列进行临时填充
                for col in feature_cols:
                    if data_copy[col].isna().sum() > 0:
                        if pd.api.types.is_numeric_dtype(data_copy[col]):
                            if method == 'mean':
                                data_copy[col] = data_copy[col].fillna(data_copy[col].mean())
                            elif method == 'median':
                                data_copy[col] = data_copy[col].fillna(data_copy[col].median())
                            elif method == 'mode':
                                mode_value = data_copy[col].mode().iloc[0] if not data_copy[col].mode().empty else 0
                                data_copy[col] = data_copy[col].fillna(mode_value)
                        else:
                            # 分类列使用众数填充
                            mode_value = data_copy[col].mode().iloc[0] if not data_copy[col].mode().empty else 'unknown'
                            data_copy[col] = data_copy[col].fillna(mode_value)
                
                # 特征编码
                encoded_features = []
                for col in feature_cols:
                    # 先检查是否是分类列、对象列或字符串列
                    if pd.api.types.is_categorical_dtype(data_copy[col]) or pd.api.types.is_object_dtype(data_copy[col]) or str(data_copy[col].dtype) == 'string' or str(data_copy[col].dtype) == 'str':
                        # 分类列编码
                        le = LabelEncoder()
                        try:
                            # 确保所有值都是字符串
                            str_col = data_copy[col].astype(str)
                            # 处理可能的缺失值
                            str_col = str_col.fillna('unknown')
                            encoded_col = le.fit_transform(str_col)
                            # 确保是numpy数组后再reshape
                            encoded_col = np.array(encoded_col)
                            encoded_features.append(encoded_col.reshape(-1, 1))
                        except Exception as e:
                            pass
                    # 检查是否是时间列
                    elif pd.api.types.is_datetime64_any_dtype(data_copy[col]):
                        # 时间列编码
                        try:
                            encoded_features.append(np.array(data_copy[col].dt.year).reshape(-1, 1))
                            encoded_features.append(np.array(data_copy[col].dt.month).reshape(-1, 1))
                            encoded_features.append(np.array(data_copy[col].dt.day).reshape(-1, 1))
                            if hasattr(data_copy[col].dt, 'hour'):
                                encoded_features.append(np.array(data_copy[col].dt.hour).reshape(-1, 1))
                            else:
                                encoded_features.append(np.zeros((len(data_copy), 1)))
                        except Exception as e:
                            pass
                    # 否则认为是数值列
                    else:
                        try:
                            # 确保是numpy数组后再reshape
                            col_values = np.array(data_copy[col], dtype=np.float64)
                            # 处理可能的NaN值
                            col_values = np.nan_to_num(col_values, nan=0.0)
                            encoded_features.append(col_values.reshape(-1, 1))
                        except Exception as e:
                            pass
                
                if not encoded_features:
                    continue
                
                # 合并特征
                X = np.hstack(encoded_features)
                y = data_copy[target_col].values
                
                # 多项式特征
                poly = PolynomialFeatures(degree=poly_degree, include_bias=False)
                X_poly = poly.fit_transform(X)
                
                # 划分训练集和验证集
                X_train, X_val, y_train, y_val = train_test_split(
                    X_poly[train_mask], y[train_mask], test_size=0.2, random_state=42
                )
                
                # 训练模型
                model = LinearRegression()
                model.fit(X_train, y_train)
                
                # 在验证集上评估模型性能
                val_predictions = model.predict(X_val)
                mse = mean_squared_error(y_val, val_predictions)
                r2 = r2_score(y_val, val_predictions)
                all_val_scores.append(r2)
                
                # 计算权重（MSE越小，权重越大）
                weight = 1.0 / (mse + 1e-10)  # 避免除以零
                method_weights.append(weight)
                
                # 预测
                if test_mask.sum() > 0:
                    predictions = model.predict(X_poly[test_mask])
                    all_predictions.append(predictions)
            
            # 检查是否有有效预测
            if all_predictions:
                # 归一化权重
                total_weight = sum(method_weights)
                normalized_weights = [w / total_weight for w in method_weights]
                
                # 计算加权平均
                weighted_predictions = np.zeros_like(all_predictions[0])
                for pred, weight in zip(all_predictions, normalized_weights):
                    weighted_predictions += pred * weight
                
                # 计算平均验证分数
                avg_val_score = np.mean(all_val_scores)
                
                # 计算预测相似度
                predictions_array = np.array(all_predictions)
                similarity = 100 - np.std(predictions_array, axis=0).mean() * 100
                if similarity < 0:
                    similarity = 0
                
                # 更新最佳结果，基于验证分数
                if avg_val_score > best_score:
                    best_score = avg_val_score
                    best_predictions = weighted_predictions
                    best_weights = normalized_weights
                
                # 调整迭代次数
                if avg_val_score < 0.6:
                    max_iterations += 5
                    ColorPrinter.print_info(f'验证分数较低，增加迭代次数到 {max_iterations}')
                elif avg_val_score > 0.85:
                    max_iterations -= 5
                    ColorPrinter.print_info(f'验证分数较高，减少迭代次数到 {max_iterations}')
                
                # 检查迭代次数范围
                if max_iterations < 5 or max_iterations > 1000:
                    ColorPrinter.print_info(f'迭代次数超出范围 ({max_iterations})，停止迭代')
                    # 使用当前最佳计算结果
                    ColorPrinter.print_info('使用当前最佳计算结果')
                    break
            
            current_iteration += 1
        
        # 使用最佳预测结果
        if best_predictions is not None:
            data.loc[test_mask, target_col] = best_predictions
            ColorPrinter.print_info(f'EI填充列 {target_col}，填充了 {len(best_predictions)} 个缺失值')
            ColorPrinter.print_info(f'填充方法权重: mean={best_weights[0]:.4f}, median={best_weights[1]:.4f}, mode={best_weights[2]:.4f}')
        else:
            # 回退到均值填充
            if pd.api.types.is_numeric_dtype(data[target_col]):
                data[target_col] = data[target_col].fillna(data[target_col].mean())
            else:
                mode_value = data[target_col].mode().iloc[0] if not data[target_col].mode().empty else None
                if mode_value is not None:
                    data[target_col] = data[target_col].fillna(mode_value)
            ColorPrinter.print_info(f'EI填充失败，回退到默认填充方法')
        
        return data


"""
参数说明文档:

shortcut_nan 参数字典:
    - nan_missing_process_choice: 缺失值处理方式选择
      类型: 列表的列表
      可选值: 'fill_nan_mean', 'fill_nan_median', 'fill_nan_mode', 'fill_nan_forward', 'fill_nan_backward', 'fill_nan_intelligent', 'drop_nan', 'drop_col_high_missing', 'fill_nan_knn', 'skip'
      默认值: 无（必须指定）
      说明: 为每列指定处理方法，列表长度必须与nan_processing_col相同
        - 'fill_nan_mean': 使用均值填充缺失值
        - 'fill_nan_median': 使用中位数填充缺失值
        - 'fill_nan_mode': 使用众数填充缺失值
        - 'fill_nan_forward': 使用前向填充缺失值
        - 'fill_nan_backward': 使用后向填充缺失值
        - 'fill_nan_intelligent': 使用智能填充缺失值
        - 'drop_col_high_missing': 删除缺失率过高的列
        - 'fill_nan_knn': 使用KNN算法填充缺失值
        - 'fill_nan_ei': 使用集成填充算法填充缺失值
        - 'specify_fill': 使用用户指定的值填充缺失值
        - 'skip': 跳过该列的缺失值处理
    
    - nan_processing_col: 要处理的列
      类型: 列表
      可选值: 数据集中的列名
      默认值: 无（必须指定）
      说明: 指定要进行缺失值处理的列，列表长度必须与nan_missing_process_choice相同
    
    - drop_threshold: 缺失值比例阈值
      类型: 浮点数
      可选值: 0.0 - 1.0
      默认值: 0.5
      说明: 当列的缺失值比例超过此阈值时，删除该列（仅在drop_col_high_missing模式下使用）
    
    - knn_n_neighbors: KNN算法的邻居数
      类型: 整数
      可选值: >= 1
      默认值: 5
      说明: KNN填充时使用的邻居数量（仅在fill_nan_knn模式下使用）

使用示例:
    # 为不同列应用不同的缺失值处理方法
    nan_params = {
        'nan_missing_process_choice': [
            ['drop_nan', 'fill_nan_knn'],      # 数值列1的处理方法
            ['drop_nan', 'drop_col_high_missing', 'fill_nan_knn'],  # 数值列2的处理方法
            ['drop_nan', 'fill_nan_mode'],      # 分类列的处理方法
            ['drop_nan', 'skip']                # 无缺失值列的处理方法
        ],
        'nan_processing_col': ['numeric_col1', 'numeric_col2', 'categorical_col', 'datetime_col'],
        'drop_threshold': 0.4,
        'knn_n_neighbors': 5
    }
    nan_processor = NanPreprocessing(shortcut_nan=nan_params)
    cleaned_data = nan_processor(data, 'normal', 'label')
"""


if __name__ == "__main__":
    import pandas as pd
    import numpy as np
    
    ColorPrinter.print_any("=" * 60)
    ColorPrinter.print_info("缺失值处理测试程序")
    ColorPrinter.print_any("=" * 60)
    
    # 创建包含缺失值的测试数据
    np.random.seed(42)
    test_data = pd.DataFrame({
        'feature1': [1, 2, np.nan, 4, 5, np.nan, 7, 8, 9, 10],
        'feature2': [np.nan, 12, 13, np.nan, 15, 16, np.nan, 18, 19, 20],
        'feature3': [21, 22, 23, 24, 25, 26, 27, 28, np.nan, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    ColorPrinter.print_any("\n原始数据:")
    ColorPrinter.print_any(test_data)
    ColorPrinter.print_any("\n缺失值统计:")
    ColorPrinter.print_any(test_data.isnull().sum())
    
    # 测试不同的缺失值处理方式
    test_cases = [
        {
            'nan_missing_process_choice': 'fill_nan_mean',
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'fill_nan_median',
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'drop_nan',
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'drop_col_high_missing',
            'drop_threshold': 0.3
        },
        {
            'nan_missing_process_choice': ['fill_nan_mean', 'drop_col_high_missing'],
            'drop_threshold': 0.5
        },
        {
            'nan_missing_process_choice': 'fill_nan_knn',
            'drop_threshold': 0.5,
            'knn_n_neighbors': 3
        }
    ]
    
    for i, params in enumerate(test_cases, 1):
        ColorPrinter.print_any(f"\n{'=' * 60}")
        ColorPrinter.print_info(f"测试 {i}: {params['nan_missing_process_choice']}")
        ColorPrinter.print_any('=' * 60)
        
        nan_processor = NanPreprocessing(shortcut_nan=params)
        cleaned_data = nan_processor(test_data.copy(), 'normal', 'label')
        
        ColorPrinter.print_any("\n处理后的数据:")
        ColorPrinter.print_any(cleaned_data)
        ColorPrinter.print_any("\n处理后缺失值统计:")
        ColorPrinter.print_any(cleaned_data.isnull().sum())
    
    # 全面渗透测试套件
    ColorPrinter.print_any(f"\n{'=' * 80}")
    ColorPrinter.print_info("全面渗透测试套件")
    ColorPrinter.print_any('=' * 80)
    
    # 测试1: 列级参数测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试1: 列级参数测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_col_level = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_col_level = {
        'nan_missing_process_choice': [['fill_nan_mean'], ['fill_nan_median'], ['fill_nan_mode']],
        'nan_processing_col': ['col1', 'col2', 'col3'],
        'drop_threshold': 0.5
    }
    
    nan_processor_col_level = NanPreprocessing(shortcut_nan=params_col_level)
    result_col_level = nan_processor_col_level(test_data_col_level.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_col_level)
    ColorPrinter.print_any(f"\n数据形状变化: {test_data_col_level.shape} -> {result_col_level.shape}")
    ColorPrinter.print_info("期望：col1均值填充，col2中位数填充，col3众数填充")
    
    # 测试2: 前向和后向填充测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试2: 前向和后向填充测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_forward_backward = pd.DataFrame({
        'col1': [1, np.nan, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, 14, np.nan, np.nan, 17, 18, 19, 20],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_forward_backward = {
        'nan_missing_process_choice': [['fill_nan_forward'], ['fill_nan_backward']],
        'nan_processing_col': ['col1', 'col2'],
        'drop_threshold': 0.5
    }
    
    nan_processor_forward_backward = NanPreprocessing(shortcut_nan=params_forward_backward)
    result_forward_backward = nan_processor_forward_backward(test_data_forward_backward.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_forward_backward)
    ColorPrinter.print_info("期望：col1前向填充，col2后向填充")
    
    # 测试3: 智能填充测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试3: 智能填充测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_intelligent = pd.DataFrame({
        'col1': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
        'col2': [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.1],
        'col3': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_intelligent = {
        'nan_missing_process_choice': 'fill_nan_intelligent',
        'drop_threshold': 0.5
    }
    
    nan_processor_intelligent = NanPreprocessing(shortcut_nan=params_intelligent)
    result_intelligent = nan_processor_intelligent(test_data_intelligent.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_intelligent)
    ColorPrinter.print_info("期望：col3使用col1或col2进行智能填充")
    
    # 测试4: 删除高缺失值列测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试4: 删除高缺失值列测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_drop_col = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, np.nan, 14, 15, np.nan, 17, 18, np.nan, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_drop_col = {
        'nan_missing_process_choice': 'drop_col_high_missing',
        'drop_threshold': 0.3
    }
    
    nan_processor_drop_col = NanPreprocessing(shortcut_nan=params_drop_col)
    result_drop_col = nan_processor_drop_col(test_data_drop_col.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_drop_col)
    ColorPrinter.print_any(f"\n数据形状变化: {test_data_drop_col.shape} -> {result_drop_col.shape}")
    ColorPrinter.print_info("期望：删除col2（缺失值比例33.3% > 30%阈值）")
    
    # 测试5: KNN填充测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试5: KNN填充测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_knn = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_knn = {
        'nan_missing_process_choice': 'fill_nan_knn',
        'drop_threshold': 0.5,
        'knn_n_neighbors': 3
    }
    
    nan_processor_knn = NanPreprocessing(shortcut_nan=params_knn)
    result_knn = nan_processor_knn(test_data_knn.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_knn)
    ColorPrinter.print_info("期望：col1和col2使用KNN填充缺失值")
    
    # 测试6: 组合处理测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试6: 组合处理测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_combined = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_combined = {
        'nan_missing_process_choice': [['fill_nan_mean', 'drop_col_high_missing'], ['fill_nan_median', 'skip']],
        'nan_processing_col': ['col1', 'col2', 'col3'],
        'drop_threshold': 0.3
    }
    
    nan_processor_combined = NanPreprocessing(shortcut_nan=params_combined)
    result_combined = nan_processor_combined(test_data_combined.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_combined)
    ColorPrinter.print_info("期望：col1均值填充，col2中位数填充，col3删除（如果缺失值比例高）")
    
    # 测试7: 边界情况测试 - 无缺失值
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试7: 边界情况测试 - 无缺失值")
    ColorPrinter.print_any('-' * 60)
    
    test_data_no_missing = pd.DataFrame({
        'col1': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_no_missing = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_no_missing = NanPreprocessing(shortcut_nan=params_no_missing)
    result_no_missing = nan_processor_no_missing(test_data_no_missing.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_no_missing)
    ColorPrinter.print_info("期望：数据保持不变，无缺失值处理")
    
    # 测试8: 边界情况测试 - 全部缺失值
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试8: 边界情况测试 - 全部缺失值")
    ColorPrinter.print_any('-' * 60)
    
    test_data_all_missing = pd.DataFrame({
        'col1': [np.nan, np.nan, np.nan, np.nan, np.nan],
        'col2': [np.nan, np.nan, np.nan, np.nan, np.nan],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    params_all_missing = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_all_missing = NanPreprocessing(shortcut_nan=params_all_missing)
    result_all_missing = nan_processor_all_missing(test_data_all_missing.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_all_missing)
    ColorPrinter.print_info("期望：均值填充，但可能无法计算均值")
    
    # 测试9: 异常情况测试 - 空数据
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试9: 异常情况测试 - 空数据")
    ColorPrinter.print_any('-' * 60)
    
    test_data_empty = pd.DataFrame({
        'col1': [],
        'col2': [],
        'label': []
    })
    
    params_empty = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    try:
        nan_processor_empty = NanPreprocessing(shortcut_nan=params_empty)
        result_empty = nan_processor_empty(test_data_empty.copy(), 'normal', 'label')
        
        ColorPrinter.print_any("\n处理结果:")
        ColorPrinter.print_any(result_empty)
        ColorPrinter.print_info("期望：空数据保持不变")
    except Exception as e:
        ColorPrinter.print_error(f"异常处理: {e}")
        ColorPrinter.print_info("期望：空数据应该能正常处理")
    
    # 测试10: 不同数据类型测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试10: 不同数据类型测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_types = pd.DataFrame({
        'numeric_col': [1, 2, np.nan, 4, 5],
        'string_col': ['A', 'B', np.nan, 'D', 'E'],
        'float_col': [1.1, 2.2, np.nan, 4.4, 5.5],
        'int_col': [10, 20, np.nan, 40, 50],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    params_types = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_types = NanPreprocessing(shortcut_nan=params_types)
    result_types = nan_processor_types(test_data_types.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_types)
    ColorPrinter.print_info("期望：数值列均值填充，字符串列跳过或特殊处理")
    
    # 测试11: 大数据集测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试11: 大数据集测试")
    ColorPrinter.print_any('-' * 60)
    
    np.random.seed(42)
    large_data_size = 10000
    test_data_large = pd.DataFrame({
        'feature1': np.random.randn(large_data_size),
        'feature2': np.random.randn(large_data_size),
        'feature3': np.random.randn(large_data_size),
        'label': np.random.choice(['A', 'B'], large_data_size)
    })
    
    # 随机添加缺失值
    missing_indices1 = np.random.choice(large_data_size, int(large_data_size * 0.1), replace=False)
    missing_indices2 = np.random.choice(large_data_size, int(large_data_size * 0.15), replace=False)
    test_data_large.loc[missing_indices1, 'feature1'] = np.nan
    test_data_large.loc[missing_indices2, 'feature2'] = np.nan
    
    params_large = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_large = NanPreprocessing(shortcut_nan=params_large)
    result_large = nan_processor_large(test_data_large.copy(), 'normal', 'label')
    
    ColorPrinter.print_any(f"\n处理结果（数据大小: {large_data_size} 行）:")
    ColorPrinter.print_any(f"原始缺失值: {test_data_large.isnull().sum().sum()}")
    ColorPrinter.print_any(f"处理后缺失值: {result_large.isnull().sum().sum()}")
    ColorPrinter.print_info("期望：所有缺失值被填充")
    
    # 测试12: 极端阈值测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试12: 极端阈值测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_threshold = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, np.nan, 14, 15, np.nan, 17, 18, np.nan, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_threshold_low = {
        'nan_missing_process_choice': 'drop_col_high_missing',
        'drop_threshold': 0.1  # 极低阈值
    }
    
    params_threshold_high = {
        'nan_missing_process_choice': 'drop_col_high_missing',
        'drop_threshold': 0.9  # 极高阈值
    }
    
    nan_processor_threshold_low = NanPreprocessing(shortcut_nan=params_threshold_low)
    result_threshold_low = nan_processor_threshold_low(test_data_threshold.copy(), 'normal', 'label')
    
    nan_processor_threshold_high = NanPreprocessing(shortcut_nan=params_threshold_high)
    result_threshold_high = nan_processor_threshold_high(test_data_threshold.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n低阈值（0.1）结果:")
    ColorPrinter.print_any(f"删除列数: {test_data_threshold.shape[1] - result_threshold_low.shape[1]}")
    ColorPrinter.print_any("高阈值（0.9）结果:")
    ColorPrinter.print_any(f"删除列数: {test_data_threshold.shape[1] - result_threshold_high.shape[1]}")
    
    # 测试13: 标签列保护测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试13: 标签列保护测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_label = pd.DataFrame({
        'feature1': [1, 2, np.nan, 4, 5],
        'feature2': [11, 12, np.nan, 14, 15],
        'label': [np.nan, 'B', np.nan, 'B', 'A']  # 标签列也有缺失值
    })
    
    params_label = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_label = NanPreprocessing(shortcut_nan=params_label)
    result_label = nan_processor_label(test_data_label.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(result_label)
    ColorPrinter.print_info("期望：标签列的缺失值不被处理")
    
    # 测试14: 多列同时处理测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试14: 多列同时处理测试")
    ColorPrinter.print_any('-' * 60)
    
    test_data_multi = pd.DataFrame({
        'col1': [1, 2, np.nan, 4, 5, 6, 7, 8, 9, 10],
        'col2': [11, 12, 13, np.nan, 15, 16, 17, 18, 19, 20],
        'col3': [21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
        'col4': [31, 32, 33, 34, 35, 36, 37, 38, 39, 40],
        'col5': [41, 42, 43, 44, 45, 46, 47, 48, 49, 50],
        'label': ['A', 'B', 'A', 'B', 'A', 'B', 'A', 'B', 'A', 'B']
    })
    
    params_multi = {
        'nan_missing_process_choice': [
            ['fill_nan_mean'], 
            ['fill_nan_median'], 
            ['fill_nan_mode'], 
            ['fill_nan_forward'], 
            ['fill_nan_backward']
        ],
        'nan_processing_col': ['col1', 'col2', 'col3', 'col4', 'col5'],
        'drop_threshold': 0.5
    }
    
    nan_processor_multi = NanPreprocessing(shortcut_nan=params_multi)
    result_multi = nan_processor_multi(test_data_multi.copy(), 'normal', 'label')
    
    ColorPrinter.print_any("\n处理结果:")
    ColorPrinter.print_any(f"数据形状: {test_data_multi.shape} -> {result_multi.shape}")
    ColorPrinter.print_info("期望：每列使用不同的填充方法")
    
    # 测试15: 性能测试
    ColorPrinter.print_any(f"\n{'-' * 60}")
    ColorPrinter.print_info("测试15: 性能测试")
    ColorPrinter.print_any('-' * 60)
    
    import time
    test_data_perf = pd.DataFrame({
        'feature1': np.random.randn(5000),
        'feature2': np.random.randn(5000),
        'feature3': np.random.randn(5000),
        'label': np.random.choice(['A', 'B'], 5000)
    })
    
    # 随机添加缺失值
    for col in ['feature1', 'feature2', 'feature3']:
        missing_indices = np.random.choice(5000, int(5000 * 0.1), replace=False)
        test_data_perf.loc[missing_indices, col] = np.nan
    
    params_perf = {
        'nan_missing_process_choice': 'fill_nan_mean',
        'drop_threshold': 0.5
    }
    
    nan_processor_perf = NanPreprocessing(shortcut_nan=params_perf)
    
    start_time = time.time()
    result_perf = nan_processor_perf(test_data_perf.copy(), 'normal', 'label')
    end_time = time.time()
    
    ColorPrinter.print_any(f"\n处理时间: {end_time - start_time:.4f} 秒")
    ColorPrinter.print_any(f"数据大小: {len(test_data_perf)} 行")
    ColorPrinter.print_any(f"原始缺失值: {test_data_perf.isnull().sum().sum()}")
    ColorPrinter.print_any(f"处理后缺失值: {result_perf.isnull().sum().sum()}")
    
    ColorPrinter.print_any("\n" + "=" * 80)
    ColorPrinter.print_success("全面渗透测试完成！")
    ColorPrinter.print_any("=" * 80)
