import sys
import os

# 添加项目根目录到Python路径
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')))

from myffe.tools.Loggerv1_1 import log_class
from myffe.tools.color_printer import ColorPrinter
from myffe.tools.evaluation_decorator import with_evaluation
from myffe.tools.data_inspector import inspect_data
from myffe.tools.param_utils import convert_to_list, get_valid_columns
import pandas as pd
import numpy as np

# 尝试导入可选依赖
try:
    from sklearn.preprocessing import LabelEncoder, OneHotEncoder
    from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
    from sklearn.decomposition import LatentDirichletAllocation
    from sklearn.feature_selection import SelectKBest, chi2
    sklearn_available = True
except ImportError:
    sklearn_available = False

try:
    import nltk
    from nltk.corpus import stopwords
    from nltk.stem import PorterStemmer
    nltk_available = True
except ImportError:
    nltk_available = False

try:
    import jieba
    jieba_available = True
except ImportError:
    jieba_available = False

@log_class(type='use')
class StrPreprocessing:
    def __init__(self, shortcut_str=None):
        self.shortcut_str = shortcut_str

    @with_evaluation
    def __call__(self, data, data_label=None):
        # 直接处理完整数据
        self.data = data.copy()
        
        # 检查是否使用 str_ways 参数（新格式）
        if self.shortcut_str and 'str_ways' in self.shortcut_str:
            str_ways = self.shortcut_str['str_ways']
            
            # 如果 str_ways 为空或所有方法都是 skip，跳过处理
            if not str_ways or all('skip' in way.get('method', []) for way in str_ways):
                ColorPrinter.print_info("跳过字符串处理")
                return self.data
            
            data_copy = self.data.copy()
            
            # 获取通用参数
            tfidf_min_df = self.shortcut_str.get('tfidf_min_df', 1)
            tfidf_max_df = self.shortcut_str.get('tfidf_max_df', 1.0)
            tfidf_max_features = self.shortcut_str.get('tfidf_max_features', None)
            lda_min_df = self.shortcut_str.get('lda_min_df', 1)
            lda_max_df = self.shortcut_str.get('lda_max_df', 1.0)
            lda_max_features = self.shortcut_str.get('lda_max_features', 1000)
            lda_n_components = self.shortcut_str.get('lda_n_components', 10)
            lda_random_state = self.shortcut_str.get('lda_random_state', 42)
            
            # 遍历每个 str_ways 配置
            for way_config in str_ways:
                str_cols = convert_to_list(way_config.get('str_cols', []), 'str_cols')
                method = way_config.get('method', 'skip')
                str_choice = method if isinstance(method, str) else (method[0] if method else 'skip')
                use_chinese_tokenizer = way_config.get('use_chinese_tokenizer', False)
                feature_selection = way_config.get('feature_selection', False)
                k_features = way_config.get('k_features', 100)
                
                if str_choice == 'skip':
                    continue
                
                if not sklearn_available:
                    ColorPrinter.print_error('scikit-learn 未安装，无法进行字符串编码处理')
                    return data_copy
                
                for col in str_cols:
                    col = col.strip()
                    if col not in data_copy.columns:
                        continue
                    
                    # 检查是否为日期列
                    if self.is_date_column(data_copy, col):
                        ColorPrinter.print_info(f'列 {col} 是日期类型，跳过字符串处理')
                        continue
                    
                    # 检查列是否为数值类型
                    if pd.api.types.is_numeric_dtype(data_copy[col]):
                        ColorPrinter.print_info(f'列 {col} 是数值类型，跳过字符串处理')
                        continue
                    
                    if str_choice == 'label_encoder':
                        le = LabelEncoder()
                        data_copy[col] = le.fit_transform(data_copy[col])
                        ColorPrinter.print_progress(f"对列 {col} 进行标签编码")
                    
                    elif str_choice == 'one_hot_encoder':
                        encoded_df = self.onehotencode(col)
                        data_copy = pd.concat([data_copy, encoded_df], axis=1)
                        data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行独热编码")
                    
                    elif str_choice == 'frequency_encoder':
                        freq = data_copy[col].value_counts(normalize=True)
                        data_copy[col] = data_copy[col].map(freq)
                        ColorPrinter.print_progress(f"对列 {col} 进行频率编码")
                    
                    elif str_choice == 'count_encoder':
                        count = data_copy[col].value_counts()
                        data_copy[col] = data_copy[col].map(count)
                        ColorPrinter.print_progress(f"对列 {col} 进行计数编码")
                    
                    elif str_choice == 'target_encoder':
                        if data_label and data_label in data_copy.columns:
                            mean_encode = data_copy.groupby(col)[data_label].mean()
                            data_copy[col] = data_copy[col].map(mean_encode)
                            ColorPrinter.print_progress(f"对列 {col} 进行目标编码")
                    
                    elif str_choice == 'tfidf':
                        tfidf_df = self.tfidf(col, min_df=tfidf_min_df, max_df=tfidf_max_df, max_features=tfidf_max_features)
                        if not tfidf_df.empty:
                            data_copy = pd.concat([data_copy, tfidf_df], axis=1)
                            data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行 TF-IDF 处理")
                    
                    elif str_choice == 'count':
                        count_df = self.count(col, min_df=tfidf_min_df, max_df=tfidf_max_df, max_features=tfidf_max_features)
                        if not count_df.empty:
                            data_copy = pd.concat([data_copy, count_df], axis=1)
                            data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行计数向量化")
                    
                    elif str_choice == 'lda':
                        lda_df = self.lda(col, min_df=lda_min_df, max_df=lda_max_df, max_features=lda_max_features, n_components=lda_n_components, random_state=lda_random_state)
                        if not lda_df.empty:
                            data_copy = pd.concat([data_copy, lda_df], axis=1)
                            data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行 LDA 处理")
                    
                    else:
                        ColorPrinter.print_warning(f"未知的字符串处理方式: {str_choice}")
            
            return data_copy
        
        # 检查是否使用旧格式 str_choice 参数
        elif self.shortcut_str and 'str_choice' in self.shortcut_str:
            # 检查是否有 str_choice 参数
            if 'str_choice' in self.shortcut_str:
                str_choice = self.shortcut_str['str_choice']
                # 使用 param_utils 工具进行列表转换
                str_cols = convert_to_list(self.shortcut_str.get('str_cols', []), 'str_cols')
                
                data_copy = self.data.copy()
                # 设置 self.data 属性，以便 onehotencode 等方法可以访问
                self.data = data_copy
                
                # 如果没有指定列，跳过处理
                if not str_cols:
                    ColorPrinter.print_warning('未指定字符串处理列，跳过字符串处理')
                    return data_copy
                
                if not sklearn_available:
                    ColorPrinter.print_error('scikit-learn 未安装，无法进行字符串编码处理')
                    return data_copy
                
                for col in str_cols:
                    col = col.strip()
                    if col not in data_copy.columns:
                        continue
                    
                    if str_choice == 'label_encoder':
                        # 标签编码
                        le = LabelEncoder()
                        data_copy[col] = le.fit_transform(data_copy[col])
                        ColorPrinter.print_progress(f"对列 {col} 进行标签编码")
                    
                    elif str_choice == 'one_hot_encoder':
                        # 独热编码
                        encoded_df = self.onehotencode(col)
                        data_copy = pd.concat([data_copy, encoded_df], axis=1)
                        data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行独热编码")
                    
                    elif str_choice == 'frequency_encoder':
                        # 频率编码
                        freq = data_copy[col].value_counts(normalize=True)
                        data_copy[col] = data_copy[col].map(freq)
                        ColorPrinter.print_progress(f"对列 {col} 进行频率编码")
                    
                    elif str_choice == 'count_encoder':
                        # 计数编码
                        count = data_copy[col].value_counts()
                        data_copy[col] = data_copy[col].map(count)
                        ColorPrinter.print_progress(f"对列 {col} 进行计数编码")
                    
                    elif str_choice == 'target_encoder':
                        # 目标编码
                        if data_label and data_label in data_copy.columns:
                            mean_encode = data_copy.groupby(col)[data_label].mean()
                            data_copy[col] = data_copy[col].map(mean_encode)
                            ColorPrinter.print_progress(f"对列 {col} 进行目标编码")
                    
                    elif str_choice == 'tfidf':
                        # TF-IDF 处理
                        tfidf_df = self.tfidf(col)
                        if not tfidf_df.empty:
                            data_copy = pd.concat([data_copy, tfidf_df], axis=1)
                            # 删除原始列
                            data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行 TF-IDF 处理")
                    
                    elif str_choice == 'count':
                        # 计数向量化
                        count_df = self.count(col)
                        if not count_df.empty:
                            data_copy = pd.concat([data_copy, count_df], axis=1)
                            # 删除原始列
                            data_copy.drop(col, axis=1, inplace=True)
                        ColorPrinter.print_progress(f"对列 {col} 进行计数向量化")
                    
                    elif str_choice == 'skip':
                        # 跳过处理
                        ColorPrinter.print_info("跳过字符串处理")
                    else:
                        # 未知的处理方式
                        ColorPrinter.print_warning(f"未知的字符串处理方式: {str_choice}")
                
                # 只返回特征列，不包含标签列
                return data_copy
            else:
                # 没有 str_choice 参数，跳过处理
                ColorPrinter.print_warning('未指定字符串处理方式，跳过字符串处理')
                return self.data
        else:
            # 没有 shortcut_str，跳过处理
            ColorPrinter.print_warning('未配置字符串处理参数，跳过字符串处理')
            return self.data
    
    def process_text_preprocessing(self, data, data_label=None, clean_ways=None):
        self.data = data
        self.clean_name = data_label
        self.clean_ways = clean_ways
        ColorPrinter.print_progress('开始文本预处理')

        # 如果没有指定清理方式，直接返回原始数据
        if not self.clean_ways:
            ColorPrinter.print_warning('未指定清理方式，跳过文本预处理')
            return self.data

        data, self.preprocessor_text = self.check_str(self.data)
        original_text_cols = list(self.preprocessor_text.columns)
        self.preprocessor_text = self.str_pre_switch(self.preprocessor_text)

        # 合并处理后的文本特征
        if not self.preprocessor_text.empty:
            data = pd.concat([data, self.preprocessor_text], axis=1)
            
            # 只删除原始文本列，保留生成的特征列
            for col in original_text_cols:
                if col in data.columns:
                    data.drop(col, axis=1, inplace=True)
                    ColorPrinter.print_progress(f"删除原始列: {col}")

        return data

    def str_pre_switch(self, text):
        preprocessor_text = text
        original_cols = list(text.columns)
        
        # 映射字符串到数字
        way_map = {
            'onehot': 1,
            'tokenize': 2,
            'count': 3,
            'tfidf': 4,
            'lda': 5,
            'stopwords': 6,
            'porter': 7,
            'enhanced': 8,
            'emotion': 9,
            'extract_emotion': 10
        }
        
        # 统一处理所有文本列，共享向量化器
        if not original_cols:
            return preprocessor_text
        
        # 预处理所有文本列
        processed_texts = {}
        for col in original_cols:
            processed_texts[col] = self.data[col].fillna('')
        
        # 针对不同处理方式进行统一处理
        for clean_num, way in enumerate(self.clean_ways):
            if clean_num >= len(original_cols):
                break
            col_name = original_cols[clean_num]
            
            # 转换字符串方式为数字
            if isinstance(way, str):
                way = way_map.get(way, way)
            
            try:
                if way == 1:
                    result = self.onehotencode(col_name)
                elif way == 2:
                    result = self.tokenizer(col_name)
                elif way == 3:
                    result = self.count(col_name)
                elif way == 4:
                    result = self.tfidf(col_name)
                elif way == 5:
                    result = self.lda(col_name)
                elif way == 6:
                    result = self.remove_stopwords_English_Chinese(col_name)
                elif way == 7:
                    result = self.tokenizer_porter(col_name)
                elif way == 8:
                    result = self.enhanced_preprocessor(col_name)
                elif way == 9:
                    result = self.emotion_analysis(col_name)
                elif way == 10:
                    result = self.extract_emotions(col_name)
                else:
                    continue
                
                # 检查结果是否为空DataFrame
                if isinstance(result, pd.DataFrame) and not result.empty:
                    preprocessor_text = pd.concat([preprocessor_text, result], axis=1)
                elif not isinstance(result, pd.DataFrame):
                    # 如果不是DataFrame，可能是文本处理结果，需要适当处理
                    pass
            except Exception as e:
                ColorPrinter.print_error(f"处理列 {col_name} 时出错: {e}")
                ColorPrinter.print_warning("跳过该列的处理")
                continue
        return preprocessor_text

    def check_str(self, data):
        # 对于pandas DataFrame，按原来的方式处理
        self.text_encode = pd.DataFrame(index=data.index)
        count = -1
        data_col = data.columns
        for col in data_col:
            if col == self.clean_name:
                # 处理标签列：将字符串标签转换为数字标签
                sample = data[col].sample(5).dropna()
                if self.is_string_list(list(sample)):
                    ColorPrinter.print_info(f'列 {col} 是字符串类型（标签列）')
                    ColorPrinter.print_progress('将标签列转换为数值类型...')
                    if sklearn_available:
                        label_encoder = LabelEncoder()
                        data[col] = label_encoder.fit_transform(data[col])
                        ColorPrinter.print_success(f'标签列 {col} 已转换为数值: {label_encoder.classes_} -> {label_encoder.transform(label_encoder.classes_)}')
                    else:
                        ColorPrinter.print_warning('scikit-learn 未安装，无法转换标签列为数值类型')
                continue
            
            # 检查是否为日期列
            if self.is_date_column(data, col):
                ColorPrinter.print_info(f'列 {col} 是日期类型，跳过字符串处理')
                continue
                
            sample = data[col].sample(5).dropna()
            if self.is_string_list(list(sample)):
                ColorPrinter.print_info(f'列 {col} 是字符串类型')
                count += 1
                can_convert = True
                # 先检查样本
                for i in sample:
                    if self.is_string(i):
                        try:
                            float(i)
                        except (ValueError, TypeError):
                            can_convert = False
                            break
                    else:
                        can_convert = False
                        break
                
                # 如果样本可以转换，再检查整个列
                if can_convert:
                    try:
                        data[col] = data[col].astype(float)
                        ColorPrinter.print_success(f'列 {col} 已转换为 float 类型')
                    except (ValueError, TypeError):
                        # 整个列无法转换，作为文本处理
                        self.text_encode[col] = data[col]
                else:
                    self.text_encode[col] = data[col]
        return data, self.text_encode
    
    def is_date_column(self, data, col):
        """
        检查列是否为日期类型
        
        Args:
            data: 数据框
            col: 列名
            
        Returns:
            bool: 是否为日期类型
        """
        # 检查列名是否包含日期相关关键词
        date_keywords = ['date', 'time', 'datetime', 'timestamp']
        col_lower = col.lower()
        for keyword in date_keywords:
            if keyword in col_lower:
                return True
        
        # 只对字符串类型的列进行日期检测
        if data[col].dtype == 'object':
            # 尝试将列转换为日期类型
            try:
                # 取前100个非空值进行测试
                sample_data = data[col].dropna().head(100)
                if len(sample_data) > 0:
                    # 尝试转换为日期
                    pd.to_datetime(sample_data, errors='raise')
                    return True
            except:
                pass
        
        return False

    def is_string_list(self, lst):
        return all(isinstance(item, str) for item in lst)

    def is_string(self, s):
        return isinstance(s, str)

    def remove_stopwords_English_Chinese(self, text_col):
        import pandas as pd
        
        # 检查 nltk 是否可用
        if not nltk_available:
            ColorPrinter.print_error('nltk 未安装，无法过滤停用词')
            return pd.DataFrame({text_col: self.data[text_col].fillna('')}, index=self.data.index)
        
        try:
            stopwords.words('english')
        except LookupError:
            ColorPrinter.print_info('下载nltk停用词库...')
            nltk.download('stopwords')

        english_stopwords = set(stopwords.words('english'))
        CHINESE_STOPWORDS = {
            '的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个',
            '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好',
            '自己', '这', '那', '他', '她', '它', '我们', '你们', '他们', '她们', '它们',
            '与', '或', '但', '而', '因为', '所以', '如果', '虽然', '然而', '但是', '不过',
            '吧', '呢', '啊', '哦', '嗯', '嘛', '呀', '哇', '唉', '喂', '嗨', '哎',
            '什么', '怎么', '为什么', '哪里', '谁', '什么时候', '多少', '几', '怎样',
            '可以', '能够', '应该', '必须', '需要', '想要', '希望', '可能', '也许',
            '这个', '那个', '这些', '那些', '这里', '那里', '这么', '那么', '这样', '那样'
        }

        text_data = self.data[text_col].fillna('')
        
        def process_text(text):
            words = text.split()
            # 使用纯Python实现过滤停用词
            filtered_words = [word for word in words if word.lower() not in english_stopwords and word not in CHINESE_STOPWORDS]
            return ' '.join(filtered_words)

        processed_text = text_data.apply(process_text)
        return pd.DataFrame({text_col: processed_text}, index=text_data.index)

    def tokenizer_porter(self, text_col):
        import pandas as pd

        # 检查 nltk 是否可用
        if not nltk_available:
            ColorPrinter.print_error('nltk 未安装，无法进行词干提取')
            return pd.DataFrame({text_col: self.data[text_col].fillna('')}, index=self.data.index)

        text_data = self.data[text_col].fillna('')
        
        porter = PorterStemmer()
        
        def process_text(text):
            words = text.split()
            stemmed_words = [porter.stem(word) for word in words]
            return ' '.join(stemmed_words)

        processed_text = text_data.apply(process_text)
        return pd.DataFrame({text_col: processed_text}, index=text_data.index)

    def tokenizer(self, text_col):
        text_data = self.data[text_col].fillna('')
        processed_text = text_data.apply(lambda x: x.split())
        return pd.DataFrame({text_col: processed_text}, index=text_data.index)
    
    def chinese_tokenizer(self, text):
        if jieba_available:
            return list(jieba.cut(text))
        else:
            ColorPrinter.print_warning('jieba库未安装，使用默认分词')
            return text.split()

    def enhanced_preprocessor(self, text_col):
        import re
        
        text_data = self.data[text_col].fillna('')
        
        def process_text(text):
            if not isinstance(text, str):
                return text
            if text is None or text == '':
                return text
            text = re.sub(r'<[^>]*>', '', text)
            text = re.sub(r'https?://\S+|www\.\S+', '', text)
            text = re.sub(r'\S+@\S+\.\S+', '', text)
            text = re.sub(r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b', '', text)
            text = re.sub(r'[^\w\s\u4e00-\u9fff.,!?;:()\-]', '', text)
            text = re.sub(r'([.,!?;:])\1+', r'\1', text)
            text = re.sub(r'\s+', ' ', text).strip()
            text = text.lower()
            return text

        processed_text = text_data.apply(process_text)
        return pd.DataFrame({text_col: processed_text}, index=text_data.index)

    def emotion_analysis(self, text_col):
        import re
        
        text_data = self.data[text_col].fillna('')
        
        def process_text(text):
            emotions = self.extract_emotions(text)
            if not emotions:
                return {"total": 0, "types": {}}

            emotion_counts = {}
            for emotion in emotions:
                if re.match(r'[:;=]-?[)(DP]', emotion):
                    emotion_type = "basic"
                elif re.match(r'[😀-🙏]', emotion):
                    emotion_type = "unicode_smileys"
                elif re.match(r'(开心|难过|生气|惊讶|害怕|厌恶)', emotion):
                    emotion_type = "chinese_emotions"
                else:
                    emotion_type = "other"

                emotion_counts[emotion_type] = emotion_counts.get(emotion_type, 0) + 1

            return {
                "total": len(emotions),
                "types": emotion_counts
            }

        processed_text = text_data.apply(process_text)
        return pd.DataFrame({text_col: processed_text}, index=text_data.index)

    def extract_emotions(self, text):
        import re
        if not self.is_string(text) or text is None or text == '':
            return []

        basic_emotions = re.findall(r'[:;=]-?[)(DP]', text)
        extended_emotions = re.findall(r'开心|难过|生气|惊讶|害怕|厌恶', text)
        unicode_emotions = re.findall(
            r"[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]", text)

        all_emotions = basic_emotions + extended_emotions + unicode_emotions

        return all_emotions

    def onehotencode(self, str_col):
        if not sklearn_available:
            ColorPrinter.print_error('scikit-learn 未安装，无法进行独热编码')
            return pd.DataFrame(index=self.data.index)

        encoder = OneHotEncoder(sparse_output=False)
        encoded_data = encoder.fit_transform(self.data[[str_col]])

        col_names = [f"{str_col}_{category}" for category in encoder.categories_[0]]

        encoded_df = pd.DataFrame(encoded_data, columns=col_names)

        return encoded_df

    # 统一处理的向量化器缓存
    _vectorizers = {}
    
    def count(self, str_col, min_df=None, max_df=None, max_features=None):
        if not sklearn_available:
            ColorPrinter.print_error('scikit-learn 未安装，无法进行计数向量化')
            return pd.DataFrame(index=self.data.index)

        tfidf_min_df = min_df if min_df is not None else self.shortcut_str.get('tfidf_min_df', 1)
        tfidf_max_df = max_df if max_df is not None else self.shortcut_str.get('tfidf_max_df', 1.0)
        tfidf_max_features = max_features if max_features is not None else self.shortcut_str.get('tfidf_max_features', None)
        use_chinese_tokenizer = self.shortcut_str.get('use_chinese_tokenizer', False)

        # 选择分词器
        tokenizer = self.chinese_tokenizer if use_chinese_tokenizer else None
        
        # 生成向量化器缓存键
        cache_key = f"count_{tfidf_min_df}_{tfidf_max_df}_{tfidf_max_features}_{use_chinese_tokenizer}"
        
        if cache_key not in self._vectorizers:
            # 创建并训练向量化器
            ColorPrinter.print_progress(f'创建并训练 CountVectorizer 向量化器')
            count = CountVectorizer(
                stop_words='english' if not use_chinese_tokenizer else None,
                max_df=tfidf_max_df,
                min_df=tfidf_min_df,
                max_features=tfidf_max_features,
                tokenizer=tokenizer
            )
            # 对所有文本列进行训练，共享词汇表
            all_texts = []
            if hasattr(self, 'preprocessor_text') and not self.preprocessor_text.empty:
                for col in self.preprocessor_text.columns:
                    all_texts.extend(self.data[col].fillna('').tolist())
            else:
                # 如果 preprocessor_text 不存在，只使用当前列进行训练
                all_texts.extend(self.data[str_col].fillna('').tolist())
            count.fit(all_texts)
            self._vectorizers[cache_key] = count
        else:
            # 使用缓存的向量化器
            ColorPrinter.print_info('使用缓存的 CountVectorizer 向量化器')
            count = self._vectorizers[cache_key]

        bag = count.transform(self.data[str_col].fillna('')).toarray()
        # 为列名添加原始列名作为前缀，避免重复
        feature_names = [f"{str_col}_{name}" for name in count.get_feature_names_out()]
        bag_df = pd.DataFrame(bag, columns=feature_names, index=self.data.index)
        
        # 特征选择
        bag_df = self.feature_selection(bag_df)
        
        return bag_df

    def tfidf(self, str_col, min_df=None, max_df=None, max_features=None):
        if not sklearn_available:
            ColorPrinter.print_error('scikit-learn 未安装，无法进行 TF-IDF 向量化')
            return pd.DataFrame(index=self.data.index)

        tfidf_max_df = max_df if max_df is not None else self.shortcut_str.get('tfidf_max_df', 1.0)
        tfidf_min_df = min_df if min_df is not None else self.shortcut_str.get('tfidf_min_df', 1)
        tfidf_max_features = max_features if max_features is not None else self.shortcut_str.get('tfidf_max_features', None)
        use_chinese_tokenizer = self.shortcut_str.get('use_chinese_tokenizer', False)

        # 选择分词器
        tokenizer = self.chinese_tokenizer if use_chinese_tokenizer else None
        
        # 生成向量化器缓存键
        cache_key = f"tfidf_{tfidf_min_df}_{tfidf_max_df}_{tfidf_max_features}_{use_chinese_tokenizer}"
        
        if cache_key not in self._vectorizers:
            # 创建并训练向量化器
            ColorPrinter.print_progress(f'创建并训练 TfidfVectorizer 向量化器')
            tfidf = TfidfVectorizer(
                stop_words='english' if not use_chinese_tokenizer else None,
                max_df=tfidf_max_df,
                min_df=tfidf_min_df,
                max_features=tfidf_max_features,
                tokenizer=tokenizer
            )
            # 对所有文本列进行训练，共享词汇表
            all_texts = []
            if hasattr(self, 'preprocessor_text') and not self.preprocessor_text.empty:
                for col in self.preprocessor_text.columns:
                    all_texts.extend(self.data[col].fillna('').tolist())
            else:
                # 如果 preprocessor_text 不存在，只使用当前列进行训练
                all_texts.extend(self.data[str_col].fillna('').tolist())
            tfidf.fit(all_texts)
            self._vectorizers[cache_key] = tfidf
        else:
            # 使用缓存的向量化器
            ColorPrinter.print_info('使用缓存的 TfidfVectorizer 向量化器')
            tfidf = self._vectorizers[cache_key]

        try:
            tfidf_matrix = tfidf.transform(self.data[str_col].fillna('')).toarray()
            # 为列名添加原始列名作为前缀，避免重复
            feature_names = [f"{str_col}_{name}" for name in tfidf.get_feature_names_out()]
            tfidf_df = pd.DataFrame(tfidf_matrix, columns=feature_names, index=self.data.index)
            
            # 特征选择
            tfidf_df = self.feature_selection(tfidf_df)
            
            return tfidf_df
        except ValueError:
            # 处理空词汇表的情况
            ColorPrinter.print_warning(f"列 {str_col} 无法进行 TF-IDF 处理，可能只包含停用词")
            return pd.DataFrame(index=self.data.index)


    def feature_selection(self, feature_df):
        """
        特征选择方法
        
        Args:
            feature_df: 特征数据框
            
        Returns:
            选择后的特征数据框
        """
        if not sklearn_available:
            ColorPrinter.print_error('scikit-learn 未安装，无法进行特征选择')
            return feature_df
        
        # 获取特征选择参数
        feature_selection = self.shortcut_str.get('feature_selection', False)
        k_features = self.shortcut_str.get('k_features', 100)
        
        if not feature_selection or len(feature_df.columns) <= k_features:
            return feature_df
        
        try:
            # 准备标签数据
            y = self.data[self.clean_name].values
            
            # 确保标签是数值类型
            if y.dtype == 'object':
                le = LabelEncoder()
                y = le.fit_transform(y)
            
            # 使用卡方检验进行特征选择
            ColorPrinter.print_progress(f'使用卡方检验进行特征选择，目标选择 {k_features} 个特征')
            selector = SelectKBest(chi2, k=min(k_features, len(feature_df.columns)))
            X_new = selector.fit_transform(feature_df, y)
            
            # 获取选择的特征列
            selected_columns = feature_df.columns[selector.get_support()]
            selected_df = pd.DataFrame(X_new, columns=selected_columns, index=feature_df.index)
            
            ColorPrinter.print_success(f"特征选择完成: 从 {len(feature_df.columns)} 个特征中选择了 {len(selected_df.columns)} 个特征")
            return selected_df
        except Exception as e:
            ColorPrinter.print_error(f"特征选择失败: {e}")
            return feature_df
    
    def lda(self, str_col):
        if not sklearn_available:
            ColorPrinter.print_error('scikit-learn 未安装，无法进行 LDA 主题模型分析')
            return pd.DataFrame(index=self.data.index)

        lda_max_df = self.shortcut_str.get('lda_max_df', 1.0)
        lda_min_df = self.shortcut_str.get('lda_min_df', 1)
        lda_max_features = self.shortcut_str.get('lda_max_features', 1000)
        lda_n_components = self.shortcut_str.get('lda_n_components', 10)
        lda_random_state = self.shortcut_str.get('lda_random_state', 42)
        use_chinese_tokenizer = self.shortcut_str.get('use_chinese_tokenizer', False)

        # 选择分词器
        tokenizer = self.chinese_tokenizer if use_chinese_tokenizer else None
        
        # 生成向量化器缓存键
        cache_key = f"lda_{lda_min_df}_{lda_max_df}_{lda_max_features}_{use_chinese_tokenizer}"
        
        if cache_key not in self._vectorizers:
            # 创建并训练向量化器
            ColorPrinter.print_progress(f'创建并训练 CountVectorizer 向量化器 (LDA)')
            count = CountVectorizer(
                stop_words='english' if not use_chinese_tokenizer else None,
                max_df=lda_max_df,
                min_df=lda_min_df,
                max_features=lda_max_features,
                tokenizer=tokenizer
            )
            # 对所有文本列进行训练，共享词汇表
            all_texts = []
            for col in self.preprocessor_text.columns:
                all_texts.extend(self.data[col].fillna('').tolist())
            count.fit(all_texts)
            self._vectorizers[cache_key] = count
        else:
            # 使用缓存的向量化器
            ColorPrinter.print_info('使用缓存的 CountVectorizer 向量化器 (LDA)')
            count = self._vectorizers[cache_key]

        try:
            count_matrix = count.transform(self.data[str_col].fillna(''))
            
            # 检查是否有词汇
            if count_matrix.shape[1] == 0:
                ColorPrinter.print_warning(f"列 {str_col} 无法进行 LDA 处理，可能只包含停用词")
                return pd.DataFrame(index=self.data.index)

            ColorPrinter.print_progress(f'训练 LDA 主题模型，主题数: {lda_n_components}')
            lda = LatentDirichletAllocation(
                n_components=lda_n_components,
                random_state=lda_random_state
            )

            lda_matrix = lda.fit_transform(count_matrix)

            lda_df = pd.DataFrame(lda_matrix, columns=[f'{str_col}_topic_{i}' for i in range(lda_n_components)], 
                                  index=self.data.index)
            ColorPrinter.print_info('\n主题关键词 (前5个):')
            feature_names = count.get_feature_names_out()
            for i, topic in enumerate(lda.components_):
                top_words = [feature_names[i] for i in topic.argsort()[:-6:-1]]
                ColorPrinter.print_info(f'主题 {i}: {top_words}')

            return lda_df
        except ValueError:
            # 处理空词汇表的情况
            ColorPrinter.print_warning(f"列 {str_col} 无法进行 LDA 处理，可能只包含停用词")
            return pd.DataFrame(index=self.data.index)


"""
Parameter documentation:

shortcut_str parameter dictionary:
    - tfidf_min_df: Frequency count/TF-IDF minimum document frequency
      Type: float
      Optional values: 0.0 - 1.0
      Default: 1
      Description: Ignore words with document frequency lower than this value
    
    - tfidf_max_df: Frequency count/TF-IDF maximum document frequency
      Type: float
      Optional values: 0.0 - 1.0
      Default: 1.0
      Description: Ignore words with document frequency higher than this value
    
    - tfidf_max_features: Frequency count/TF-IDF maximum number of features
      Type: integer
      Optional values: >= 0
      Default: None
      Description: Limit the maximum number of features
    
    - lda_max_df: LDA maximum document frequency
      Type: float
      Optional values: 0.0 - 1.0
      Default: 1.0
      Description: Ignore words with document frequency higher than this value
    
    - lda_min_df: LDA minimum document frequency
      Type: float
      Optional values: 0.0 - 1.0
      Default: 1
      Description: Ignore words with document frequency lower than this value
    
    - lda_max_features: LDA maximum number of features
      Type: integer
      Optional values: >= 0
      Default: 1000
      Description: Limit the maximum number of features
    
    - lda_n_components: LDA number of topics
      Type: integer
      Optional values: >= 1
      Default: 10
      Description: Number of topics for LDA model
    
    - lda_random_state: LDA random seed
      Type: integer
      Optional values: >= 0
      Default: 42
      Description: Used to ensure reproducibility of results
    
    - use_chinese_tokenizer: Whether to use Chinese tokenizer
      Type: boolean
      Optional values: True, False
      Default: False
      Description: Whether to use jieba for Chinese word segmentation
    
    - feature_selection: Whether to perform feature selection
      Type: boolean
      Optional values: True, False
      Default: False
      Description: Whether to use chi-square test for feature selection
    
    - k_features: Number of features for selection
      Type: integer
      Optional values: >= 1
      Default: 100
      Description: Number of features to retain after feature selection

Performance optimization notes:
    - Unified processing: All text columns share the same vectorizer to avoid repeated training
    - Caching mechanism: Vectorizers are cached based on parameters, and processing with the same parameters will reuse previous vectorizers
    - Batch training: Vectorizers are trained on all text columns at once when first used, improving efficiency
    - Memory optimization: Vectorizers are only created and trained when needed, reducing memory usage

Usage example:
    # English text processing
    str_params = {
        'tfidf_min_df': 0.01,
        'tfidf_max_df': 0.95,
        'tfidf_max_features': 1000,
        'lda_n_components': 10,
        'lda_random_state': 42
    }
    str_processor = StrPreprocessing(shortcut_str=str_params)
    processed_data, text_data = str_processor(data, 'label', [3, 4])
    
    # Chinese text processing (using Chinese tokenizer)
    str_params_chinese = {
        'tfidf_min_df': 1,
        'tfidf_max_df': 0.95,
        'tfidf_max_features': 1000,
        'use_chinese_tokenizer': True
    }
    str_processor_chinese = StrPreprocessing(shortcut_str=str_params_chinese)
    processed_data_chinese, text_data_chinese = str_processor_chinese(data, 'label', [4])
    
    # Text processing (with feature selection)
    str_params_feature_selection = {
        'tfidf_min_df': 0.01,
        'tfidf_max_df': 0.95,
        'tfidf_max_features': 2000,
        'feature_selection': True,
        'k_features': 500
    }
    str_processor_feature = StrPreprocessing(shortcut_str=str_params_feature_selection)
    processed_data_feature, text_data_feature = str_processor_feature(data, 'label', [4])
"""


if __name__ == "__main__":
    import sys
    import os
    
    # 添加父目录到Python路径
    sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
    
    # 移除Loggerv1_1的导入，直接定义一个简单的装饰器
    def log_class(type='use'):
        def decorator(cls):
            return cls
        return decorator
    
    # 重新定义StrPreprocessing类，不依赖Loggerv1_1
    class StrPreprocessing:
        def __init__(self, shortcut_str=None):
            self.shortcut_str = shortcut_str

        def __call__(self, data, data_label, clean_ways):
            import pandas as pd
            import numpy as np
            self.clean_name = data_label
            self.clean_ways = clean_ways
            self.data = data
            print('下面进行文本预处理')

            if self.clean_ways is None:
                self.clean_ways = []
            elif not isinstance(self.clean_ways, list):
                self.clean_ways = [self.clean_ways]

            # 如果没有指定清理方式，直接返回原始数据
            if not self.clean_ways:
                print('未指定清理方式，跳过文本预处理')
                return data, pd.DataFrame(index=data.index)

            data, self.preprocessor_text = self.check_str(self.data)
            self.preprocessor_text = self.str_pre_switch(self.preprocessor_text)

            return data, self.preprocessor_text

        def str_pre_switch(self, text):
            import pandas as pd
            preprocessor_text = text
            original_cols = list(text.columns)
            
            # 映射字符串到数字
            way_map = {
                'onehot': 1,
                'tokenize': 2,
                'count': 3,
                'tfidf': 4,
                'lda': 5,
                'stopwords': 6,
                'porter': 7,
                'enhanced': 8,
                'emotion': 9,
                'extract_emotion': 10
            }
            
            # 统一处理所有文本列，共享向量化器
            if not original_cols:
                return preprocessor_text
            
            # 预处理所有文本列
            processed_texts = {}
            for col in original_cols:
                processed_texts[col] = self.data[col].fillna('')
            
            # 针对不同处理方式进行统一处理
            for clean_num, way in enumerate(self.clean_ways):
                if clean_num >= len(original_cols):
                    break
                col_name = original_cols[clean_num]
                
                # 转换字符串方式为数字
                if isinstance(way, str):
                    way = way_map.get(way, way)
                
                if way == 1:
                    preprocessor_text = pd.concat([preprocessor_text, self.onehotencode(col_name)], axis=1)
                elif way == 2:
                    preprocessor_text = pd.concat([preprocessor_text, self.tokenizer(col_name)], axis=1)
                elif way == 3:
                    preprocessor_text = pd.concat([preprocessor_text, self.count(col_name)], axis=1)
                elif way == 4:
                    preprocessor_text = pd.concat([preprocessor_text, self.tfidf(col_name)], axis=1)
                elif way == 5:
                    preprocessor_text = pd.concat([preprocessor_text, self.lda(col_name)], axis=1)
                elif way == 6:
                    preprocessor_text = pd.concat([preprocessor_text, self.remove_stopwords_English_Chinese(col_name)], axis=1)
                elif way == 7:
                    preprocessor_text = pd.concat([preprocessor_text, self.tokenizer_porter(col_name)], axis=1)
                elif way == 8:
                    preprocessor_text = pd.concat([preprocessor_text, self.enhanced_preprocessor(col_name)], axis=1)
                elif way == 9:
                    preprocessor_text = pd.concat([preprocessor_text, self.emotion_analysis(col_name)], axis=1)
                elif way == 10:
                    preprocessor_text = pd.concat([preprocessor_text, self.extract_emotions(col_name)], axis=1)
            return preprocessor_text

        def check_str(self, data):
            import pandas as pd
            from sklearn.preprocessing import LabelEncoder
            self.text_encode = pd.DataFrame(index=data.index)
            count = -1
            data_col = data.columns
            for col in data_col:
                if col == self.clean_name:
                    # 处理标签列：将字符串标签转换为数字标签
                    sample = data[col].sample(5).dropna()
                    if self.is_string_list(list(sample)):
                        print('col:', col, 'is str type (label column)')
                        print('Converting label column to numeric...')
                        label_encoder = LabelEncoder()
                        data[col] = label_encoder.fit_transform(data[col])
                        print(f'Label column {col} converted to numeric: {label_encoder.classes_} -> {label_encoder.transform(label_encoder.classes_)}')
                    continue
                sample = data[col].sample(5).dropna()
                if self.is_string_list(list(sample)):
                    print('col:', col, 'is str type')
                    count += 1
                    can_convert = True
                    # 先检查样本
                    for i in sample:
                        if self.is_string(i):
                            try:
                                float(i)
                            except (ValueError, TypeError):
                                can_convert = False
                                break
                        else:
                            can_convert = False
                            break
                    
                    # 如果样本可以转换，再检查整个列
                    if can_convert:
                        try:
                            data[col] = data[col].astype(float)
                            print('col:', col, 'change to float type')
                        except (ValueError, TypeError):
                            # 整个列无法转换，作为文本处理
                            self.text_encode[col] = data[col]
                    else:
                        self.text_encode[col] = data[col]
            return data, self.text_encode

        def is_string_list(self, lst):
            return all(isinstance(item, str) for item in lst)

        def is_string(self, s):
            return isinstance(s, str)

        def remove_stopwords_English_Chinese(self, text):
            import nltk
            from nltk.corpus import stopwords

            try:
                stopwords.words('english')
            except LookupError:
                nltk.download('stopwords')

            english_stopwords = set(stopwords.words('english'))
            CHINESE_STOPWORDS = {
                '的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个',
                '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好',
                '自己', '这', '那', '他', '她', '它', '我们', '你们', '他们', '她们', '它们',
                '与', '或', '但', '而', '因为', '所以', '如果', '虽然', '然而', '但是', '不过',
                '吧', '呢', '啊', '哦', '嗯', '嘛', '呀', '哇', '唉', '喂', '嗨', '哎',
                '什么', '怎么', '为什么', '哪里', '谁', '什么时候', '多少', '几', '怎样',
                '可以', '能够', '应该', '必须', '需要', '想要', '希望', '可能', '也许',
                '这个', '那个', '这些', '那些', '这里', '那里', '这么', '那么', '这样', '那样'
            }

            words = text.split()
            filtered_words = [word for word in words if
                              word.lower() not in english_stopwords and word not in CHINESE_STOPWORDS]

            return ' '.join(filtered_words)

        def tokenizer_porter(self, text):
            from nltk.stem import PorterStemmer

            try:
                PorterStemmer()
            except LookupError:
                pass

            porter = PorterStemmer()
            words = text.split()
            stemmed_words = [porter.stem(word) for word in words]

            return ' '.join(stemmed_words)

        def tokenizer(self, text):
            return text.split()
        
        def chinese_tokenizer(self, text):
            try:
                import jieba
                return list(jieba.cut(text))
            except ImportError:
                print('警告: jieba库未安装，使用默认分词')
                return text.split()

        def enhanced_preprocessor(self, text):
            if not self.is_string(text):
                return text
            import re
            if text is None or text == '':
                return text
            text = re.sub(r'<[^>]*>', '', text)
            text = re.sub(r'https?://\S+|www\.\S+', '', text)
            text = re.sub(r'\S+@\S+\.\S+', '', text)
            text = re.sub(r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b', '', text)
            text = re.sub(r'[^\w\s\u4e00-\u9fff.,!?;:()\-]', '', text)
            text = re.sub(r'([.,!?;:])\1+', r'\1', text)
            text = re.sub(r'\s+', ' ', text).strip()
            text = text.lower()
            return text

        def emotion_analysis(self, text):
            import re
            emotions = self.extract_emotions(text)
            if not emotions:
                return {"total": 0, "types": {}}

            emotion_counts = {}
            for emotion in emotions:
                if re.match(r'[:;=]-?[)(DP]', emotion):
                    emotion_type = "basic"
                elif re.match(r'[😀-🙏]', emotion):
                    emotion_type = "unicode_smileys"
                elif re.match(r'(开心|难过|生气|惊讶|害怕|厌恶)', emotion):
                    emotion_type = "chinese_emotions"
                else:
                    emotion_type = "other"

                emotion_counts[emotion_type] = emotion_counts.get(emotion_type, 0) + 1

            return {
                "total": len(emotions),
                "types": emotion_counts
            }

        def extract_emotions(self, text):
            import re
            if not self.is_string(text) or text is None or text == '':
                return []

            basic_emotions = re.findall(r'[:;=]-?[)(DP]', text)
            extended_emotions = re.findall(r'开心|难过|生气|惊讶|害怕|厌恶', text)
            unicode_emotions = re.findall(
                r"[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF]", text)

            all_emotions = basic_emotions + extended_emotions + unicode_emotions

            return all_emotions

        def onehotencode(self, str_col):
            import pandas as pd
            from sklearn.preprocessing import OneHotEncoder

            encoder = OneHotEncoder(sparse_output=False)
            encoded_data = encoder.fit_transform(self.data[[str_col]])

            col_names = [f"{str_col}_{category}" for category in encoder.categories_[0]]

            encoded_df = pd.DataFrame(encoded_data, columns=col_names)

            return encoded_df

        # 统一处理的向量化器缓存
        _vectorizers = {}
        
        def count(self, str_col):
            import pandas as pd
            from sklearn.feature_extraction.text import CountVectorizer

            tfidf_min_df = self.shortcut_str.get('tfidf_min_df', 1)
            tfidf_max_df = self.shortcut_str.get('tfidf_max_df', 1.0)
            tfidf_max_features = self.shortcut_str.get('tfidf_max_features', None)
            use_chinese_tokenizer = self.shortcut_str.get('use_chinese_tokenizer', False)

            # 选择分词器
            tokenizer = self.chinese_tokenizer if use_chinese_tokenizer else None
            
            # 生成向量化器缓存键
            cache_key = f"count_{tfidf_min_df}_{tfidf_max_df}_{tfidf_max_features}_{use_chinese_tokenizer}"
            
            if cache_key not in self._vectorizers:
                # 创建并训练向量化器
                count = CountVectorizer(
                    stop_words='english' if not use_chinese_tokenizer else None,
                    max_df=tfidf_max_df,
                    min_df=tfidf_min_df,
                    max_features=tfidf_max_features,
                    tokenizer=tokenizer
                )
                # 对所有文本列进行训练，共享词汇表
                all_texts = []
                for col in self.preprocessor_text.columns:
                    all_texts.extend(self.data[col].fillna('').tolist())
                count.fit(all_texts)
                self._vectorizers[cache_key] = count
            else:
                # 使用缓存的向量化器
                count = self._vectorizers[cache_key]

            bag = count.transform(self.data[str_col].fillna('')).toarray()
            bag_df = pd.DataFrame(bag, columns=count.get_feature_names_out(), index=self.data.index)
            
            # 特征选择
            bag_df = self.feature_selection(bag_df)
            
            return bag_df

        def tfidf(self, str_col):
            import pandas as pd
            tfidf_max_df = self.shortcut_str.get('tfidf_max_df', 1.0)
            tfidf_min_df = self.shortcut_str.get('tfidf_min_df', 1)
            tfidf_max_features = self.shortcut_str.get('tfidf_max_features', None)
            use_chinese_tokenizer = self.shortcut_str.get('use_chinese_tokenizer', False)

            # 选择分词器
            tokenizer = self.chinese_tokenizer if use_chinese_tokenizer else None
            
            # 生成向量化器缓存键
            cache_key = f"tfidf_{tfidf_min_df}_{tfidf_max_df}_{tfidf_max_features}_{use_chinese_tokenizer}"
            
            if cache_key not in self._vectorizers:
                # 创建并训练向量化器
                from sklearn.feature_extraction.text import TfidfVectorizer
                tfidf = TfidfVectorizer(
                    stop_words='english' if not use_chinese_tokenizer else None,
                    max_df=tfidf_max_df,
                    min_df=tfidf_min_df,
                    max_features=tfidf_max_features,
                    tokenizer=tokenizer
                )
                # 对所有文本列进行训练，共享词汇表
                all_texts = []
                for col in self.preprocessor_text.columns:
                    all_texts.extend(self.data[col].fillna('').tolist())
                tfidf.fit(all_texts)
                self._vectorizers[cache_key] = tfidf
            else:
                # 使用缓存的向量化器
                tfidf = self._vectorizers[cache_key]

            try:
                tfidf_matrix = tfidf.transform(self.data[str_col].fillna('')).toarray()
                tfidf_df = pd.DataFrame(tfidf_matrix, columns=tfidf.get_feature_names_out(), index=self.data.index)
                
                # 特征选择
                tfidf_df = self.feature_selection(tfidf_df)
                
                return tfidf_df
            except ValueError:
                # 处理空词汇表的情况
                print(f"警告: 列 {str_col} 无法进行 TF-IDF 处理，可能只包含停用词")
                return pd.DataFrame(index=self.data.index)


        def feature_selection(self, feature_df):
            """
            特征选择方法
            
            Args:
                feature_df: 特征数据框
                
            Returns:
                选择后的特征数据框
            """
            import pandas as pd
            from sklearn.feature_selection import SelectKBest, chi2
            
            # 获取特征选择参数
            feature_selection = self.shortcut_str.get('feature_selection', False)
            k_features = self.shortcut_str.get('k_features', 100)
            
            if not feature_selection or len(feature_df.columns) <= k_features:
                return feature_df
            
            try:
                # 准备标签数据
                y = self.data[self.clean_name].values
                
                # 确保标签是数值类型
                if y.dtype == 'object':
                    from sklearn.preprocessing import LabelEncoder
                    le = LabelEncoder()
                    y = le.fit_transform(y)
                
                # 使用卡方检验进行特征选择
                selector = SelectKBest(chi2, k=min(k_features, len(feature_df.columns)))
                X_new = selector.fit_transform(feature_df, y)
                
                # 获取选择的特征列
                selected_columns = feature_df.columns[selector.get_support()]
                selected_df = pd.DataFrame(X_new, columns=selected_columns, index=feature_df.index)
                
                print(f"特征选择: 从 {len(feature_df.columns)} 个特征中选择了 {len(selected_df.columns)} 个特征")
                return selected_df
            except Exception as e:
                print(f"特征选择失败: {e}")
                return feature_df
        
        def lda(self, str_col):
            import pandas as pd
            from sklearn.feature_extraction.text import CountVectorizer

            lda_max_df = self.shortcut_str.get('lda_max_df', 1.0)
            lda_min_df = self.shortcut_str.get('lda_min_df', 1)
            lda_max_features = self.shortcut_str.get('lda_max_features', 1000)
            lda_n_components = self.shortcut_str.get('lda_n_components', 10)
            lda_random_state = self.shortcut_str.get('lda_random_state', 42)
            use_chinese_tokenizer = self.shortcut_str.get('use_chinese_tokenizer', False)

            # 选择分词器
            tokenizer = self.chinese_tokenizer if use_chinese_tokenizer else None
            
            # 生成向量化器缓存键
            cache_key = f"lda_{lda_min_df}_{lda_max_df}_{lda_max_features}_{use_chinese_tokenizer}"
            
            if cache_key not in self._vectorizers:
                # 创建并训练向量化器
                count = CountVectorizer(
                    stop_words='english' if not use_chinese_tokenizer else None,
                    max_df=lda_max_df,
                    min_df=lda_min_df,
                    max_features=lda_max_features,
                    tokenizer=tokenizer
                )
                # 对所有文本列进行训练，共享词汇表
                all_texts = []
                for col in self.preprocessor_text.columns:
                    all_texts.extend(self.data[col].fillna('').tolist())
                count.fit(all_texts)
                self._vectorizers[cache_key] = count
            else:
                # 使用缓存的向量化器
                count = self._vectorizers[cache_key]

            try:
                count_matrix = count.transform(self.data[str_col].fillna(''))
                
                # 检查是否有词汇
                if count_matrix.shape[1] == 0:
                    print(f"警告: 列 {str_col} 无法进行 LDA 处理，可能只包含停用词")
                    return pd.DataFrame(index=self.data.index)

                from sklearn.decomposition import LatentDirichletAllocation
                lda = LatentDirichletAllocation(
                    n_components=lda_n_components,
                    random_state=lda_random_state
                )

                lda_matrix = lda.fit_transform(count_matrix)

                lda_df = pd.DataFrame(lda_matrix, columns=[f'{str_col}_topic_{i}' for i in range(lda_n_components)], index=self.data.index)
                print(f'\n主题关键词 (前5个):')
                feature_names = count.get_feature_names_out()
                for i, topic in enumerate(lda.components_):
                    top_words = [feature_names[i] for i in topic.argsort()[:-6:-1]]
                    print(f'主题 {i}: {top_words}')

                return lda_df
            except ValueError:
                # 处理空词汇表的情况
                print(f"警告: 列 {str_col} 无法进行 LDA 处理，可能只包含停用词")
                return pd.DataFrame(index=self.data.index)
    
    # 测试代码
    import pandas as pd
    import numpy as np
    
    print("=" * 60)
    print("文本预处理测试程序")
    print("=" * 60)
    
    # 创建包含文本的测试数据
    test_data = pd.DataFrame({
        'text_col1': [
            'This is a sample text about machine learning',
            'Data science is an exciting field',
            'Natural language processing is important',
            'Python is a great programming language',
            'Deep learning models are powerful'
        ],
        'text_col2': [
            'Machine learning is fascinating',
            'Data analysis is essential',
            'Artificial intelligence is the future',
            'Programming is fun',
            'Big data is everywhere'
        ],
        'numeric_col': [1, 2, 3, 4, 5],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    print("\n原始数据:")
    print(test_data)
    
    # 测试不同的文本处理方式
    test_cases = [
        {
            'clean_ways': [3, 4],  # 对两个文本列分别使用词频统计和TF-IDF
            'description': '多列文本处理'
        },
        {
            'clean_ways': [4, 4],  # 对两个文本列都使用TF-IDF
            'description': '多列TF-IDF处理'
        },
        {
            'clean_ways': [],
            'description': '跳过文本处理'
        }
    ]
    
    # 测试中文文本处理
    print(f"\n{'=' * 60}")
    print("测试中文文本处理")
    print('=' * 60)
    
    chinese_test_data = pd.DataFrame({
        'text_col1': [
            '这是一段中文文本',
            '特征工程非常重要',
            '机器学习是一个热门领域',
            '数据挖掘需要大量数据',
            '人工智能正在改变世界'
        ],
        'text_col2': [
            '中文分词是文本处理的基础',
            '自然语言处理技术发展迅速',
            '深度学习在NLP中应用广泛',
            '文本分类是常见任务',
            '情感分析有很多应用场景'
        ],
        'numeric_col': [1, 2, 3, 4, 5],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    print("\n中文测试数据:")
    print(chinese_test_data)
    
    # 测试中文分词
    chinese_params = {
        'tfidf_min_df': 1,
        'tfidf_max_df': 0.95,
        'tfidf_max_features': 100,
        'use_chinese_tokenizer': True
    }
    
    str_processor_chinese = StrPreprocessing(shortcut_str=chinese_params)
    try:
        processed_data_chinese, text_data_chinese = str_processor_chinese(chinese_test_data.copy(), 'label', [4, 4])
        
        print("\n中文文本处理结果:")
        print(f"处理后数据形状: {processed_data_chinese.shape}")
        if hasattr(text_data_chinese, 'shape'):
            print(f"文本特征数据形状: {text_data_chinese.shape}")
            print("\n文本特征数据（前5行）:")
            print(text_data_chinese.head())
    except Exception as e:
        print(f"\n处理出错: {e}")
    
    # 测试特征选择
    print(f"\n{'=' * 60}")
    print("测试特征选择")
    print('=' * 60)
    
    feature_selection_params = {
        'tfidf_min_df': 1,
        'tfidf_max_df': 0.95,
        'tfidf_max_features': 50,
        'feature_selection': True,
        'k_features': 10
    }
    
    str_processor_feature = StrPreprocessing(shortcut_str=feature_selection_params)
    try:
        processed_data_feature, text_data_feature = str_processor_feature(test_data.copy(), 'label', [4, 4])
        
        print("\n特征选择结果:")
        print(f"处理后数据形状: {processed_data_feature.shape}")
        if hasattr(text_data_feature, 'shape'):
            print(f"文本特征数据形状: {text_data_feature.shape}")
            print("\n文本特征数据（前5行）:")
            print(text_data_feature.head())
    except Exception as e:
        print(f"\n处理出错: {e}")
    
    for i, test_case in enumerate(test_cases, 1):
        print(f"\n{'=' * 60}")
        print(f"测试 {i}: {test_case['description']}")
        print('=' * 60)
        
        str_params = {
            'tfidf_min_df': 1,
            'tfidf_max_df': 1.0,
            'tfidf_max_features': 10,
            'lda_n_components': 2,
            'lda_random_state': 42
        }
        
        str_processor = StrPreprocessing(shortcut_str=str_params)
        try:
            processed_data, text_data = str_processor(test_data.copy(), 'label', test_case['clean_ways'])
            
            print("\n处理后的数据形状:")
            print(f"原始数据: {test_data.shape}")
            print(f"处理后数据: {processed_data.shape}")
            
            if hasattr(text_data, 'shape'):
                print(f"文本特征数据: {text_data.shape}")
                print("\n文本特征数据（前5行）:")
                print(text_data.head())
        except Exception as e:
            print(f"\n处理出错: {e}")
    
    print("\n" + "=" * 60)
    print("测试完成！")
    print("=" * 60)
    
    # 深度渗透测试套件
    print(f"\n{'=' * 80}")
    print("深度渗透测试套件")
    print('=' * 80)
    
    # 创建更大的文本测试数据
    np.random.seed(42)
    large_text_data = pd.DataFrame({
        'text_col1': [
            f'Sample text {i} about machine learning and data science' 
            for i in range(100)
        ],
        'text_col2': [
            f'Another text {i} with different content for testing' 
            for i in range(100)
        ],
        'text_col3': [
            f'NLP is interesting {i}' 
            for i in range(100)
        ],
        'numeric_col1': np.random.randint(1, 100, 100),
        'numeric_col2': np.random.normal(50, 10, 100),
        'label': ['A'] * 50 + ['B'] * 50
    })
    
    # 测试1: 边界条件测试 - 空参数
    print(f"\n{'-' * 60}")
    print("测试1: 边界条件测试 - 空参数")
    print('-' * 60)
    
    str_processor_1 = StrPreprocessing(shortcut_str={})
    try:
        result_1, text_result_1 = str_processor_1(large_text_data.copy(), 'label', [])
        print(f"\n数据形状变化: {large_text_data.shape} -> {result_1.shape}")
        print("期望：数据保持不变")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试2: 边界条件测试 - 没有文本列
    print(f"\n{'-' * 60}")
    print("测试2: 边界条件测试 - 没有文本列")
    print('-' * 60)
    
    no_text_data = pd.DataFrame({
        'num1': np.random.randint(1, 100, 100),
        'num2': np.random.normal(50, 10, 100),
        'label': ['A'] * 50 + ['B'] * 50
    })
    
    str_processor_2 = StrPreprocessing(shortcut_str={})
    try:
        result_2, text_result_2 = str_processor_2(no_text_data.copy(), 'label', [4])
        print(f"\n数据形状变化: {no_text_data.shape} -> {result_2.shape}")
        print("期望：数据保持不变")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试3: 边界条件测试 - 空数据
    print(f"\n{'-' * 60}")
    print("测试3: 边界条件测试 - 空数据")
    print('-' * 60)
    
    empty_data = pd.DataFrame({
        'text_col': [],
        'numeric_col': [],
        'label': []
    })
    
    str_processor_3 = StrPreprocessing(shortcut_str={})
    try:
        result_3, text_result_3 = str_processor_3(empty_data.copy(), 'label', [4])
        print(f"\n数据形状变化: {empty_data.shape} -> {result_3.shape}")
        print("期望：空数据保持不变")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试4: 边界条件测试 - 所有文本处理方式
    print(f"\n{'-' * 60}")
    print("测试4: 边界条件测试 - 所有文本处理方式")
    print('-' * 60)
    
    test_ways = [
        {'ways': [1], 'desc': '独热编码'},
        {'ways': [2], 'desc': '分词'},
        {'ways': [3], 'desc': '词频统计'},
        {'ways': [4], 'desc': 'TF-IDF'},
        {'ways': [5], 'desc': 'LDA'},
        {'ways': [6], 'desc': '停用词过滤'},
        {'ways': [7], 'desc': '词干提取'},
        {'ways': [8], 'desc': '增强预处理'},
    ]
    
    for tc in test_ways:
        str_processor = StrPreprocessing(shortcut_str={})
        try:
            result, text_result = str_processor(large_text_data.copy(), 'label', tc['ways'])
            print(f"\n{tc['desc']}:")
            print(f"  数据形状变化: {large_text_data.shape} -> {result.shape}")
        except Exception as e:
            print(f"\n{tc['desc']} 出错: {e}")
    
    # 测试5: 边界条件测试 - 空字符串
    print(f"\n{'-' * 60}")
    print("测试5: 边界条件测试 - 空字符串")
    print('-' * 60)
    
    empty_str_data = pd.DataFrame({
        'text_col': [''] * 20 + ['valid text content here'] * 30,
        'numeric_col': np.random.randint(1, 100, 50),
        'label': ['A'] * 25 + ['B'] * 25
    })
    
    str_processor_5 = StrPreprocessing(shortcut_str={})
    try:
        result_5, text_result_5 = str_processor_5(empty_str_data.copy(), 'label', [4])
        print(f"\n数据形状变化: {empty_str_data.shape} -> {result_5.shape}")
        print("期望：处理应该正常进行")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试6: 边界条件测试 - 只包含停用词的文本
    print(f"\n{'-' * 60}")
    print("测试6: 边界条件测试 - 只包含停用词的文本")
    print('-' * 60)
    
    stopword_only_data = pd.DataFrame({
        'text_col': ['the a is are was were'] * 20 + ['machine learning is fun'] * 30,
        'numeric_col': np.random.randint(1, 100, 50),
        'label': ['A'] * 25 + ['B'] * 25
    })
    
    str_processor_6 = StrPreprocessing(shortcut_str={})
    try:
        result_6, text_result_6 = str_processor_6(stopword_only_data.copy(), 'label', [4])
        print(f"\n数据形状变化: {stopword_only_data.shape} -> {result_6.shape}")
        print("期望：只包含停用词的文本应该被过滤")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试7: 边界条件测试 - 数值列作为文本处理
    print(f"\n{'-' * 60}")
    print("测试7: 边界条件测试 - 数值列作为文本处理")
    print('-' * 60)
    
    numeric_as_text_data = pd.DataFrame({
        'text_col': ['1', '2', '3', '4', '5'] * 10,
        'numeric_col': [1.1, 2.2, 3.3, 4.4, 5.5] * 10,
        'label': ['A'] * 25 + ['B'] * 25
    })
    
    str_processor_7 = StrPreprocessing(shortcut_str={})
    try:
        result_7, text_result_7 = str_processor_7(numeric_as_text_data.copy(), 'label', [4])
        print(f"\n数据形状变化: {numeric_as_text_data.shape} -> {result_7.shape}")
        print("期望：数值应该被转换为浮点数而不是作为文本处理")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试8: 组合测试 - 多列组合处理
    print(f"\n{'-' * 60}")
    print("测试8: 组合测试 - 多列组合处理")
    print('-' * 60)
    
    str_processor_8 = StrPreprocessing(shortcut_str={})
    try:
        result_8, text_result_8 = str_processor_8(large_text_data.copy(), 'label', [3, 4, 5])
        print(f"\n数据形状变化: {large_text_data.shape} -> {result_8.shape}")
        print("期望：三列分别使用词频、TF-IDF和LDA处理")
    except Exception as e:
        print(f"处理出错: {e}")
    
    # 测试9: 验证结果
    print(f"\n{'-' * 60}")
    print("测试9: 验证结果")
    print('-' * 60)
    
    verify_data = pd.DataFrame({
        'text_col': [
            'machine learning is great',
            'data science uses algorithms',
            'deep learning is part of AI',
            'NLP processes text data',
            'statistics is fundamental'
        ],
        'label': ['A', 'B', 'A', 'B', 'A']
    })
    
    str_processor_9 = StrPreprocessing(shortcut_str={})
    try:
        result_9, text_result_9 = str_processor_9(verify_data.copy(), 'label', [4])
        print(f"\n原始数据形状: {verify_data.shape}")
        print(f"处理后数据形状: {result_9.shape}")
        print(f"文本特征形状: {text_result_9.shape}")
        print("期望：成功提取文本特征")
    except Exception as e:
        print(f"处理出错: {e}")
    
    print(f"\n{'=' * 80}")
    print("深度渗透测试完成！")
    print('=' * 80)
