import sys
import os

# 添加项目根目录到Python路径
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', '..')))

import pandas as pd
import numpy as np
from myffe.tools.interactive import interactive
from myffe.tools.visualize_route import visualize_route
from myffe.tools.color_printer import ColorPrinter
from myffe.tools.param_utils import convert_to_list, ensure_list



def analyze_string_distribution(data):
    """分析数据的字符串分布，推荐适合的字符串处理方法
    
    Args:
        data: pandas DataFrame，输入数据
    
    Returns:
        list: 包含每个字符串列处理参数的字典列表
        list: 字符串列列表
    """
    # 找出字符串列
    string_columns = []
    for col in data.columns:
        # 跳过数值列
        if pd.api.types.is_numeric_dtype(data[col]):
            continue
        # 检查是否是字符串列
        if data[col].dtype == 'object' or data[col].dtype == 'string' or str(data[col].dtype) == 'str':
            # 检查是否是纯字符串列（非数值）
            try:
                pd.to_numeric(data[col])
            except ValueError:
                string_columns.append(col)
    
    # 为每个字符串列生成单独的参数配置
    column_params_list = []
    
    if string_columns:
        for col in string_columns:
            col_data = data[col].astype(str)
            unique_count = col_data.nunique()
            # 处理可能的NaN值，确保所有值都是字符串
            lengths = col_data.apply(lambda x: len(x) if isinstance(x, str) else 0)
            col_avg_length = lengths.mean()
            sample_count = len(col_data)
            
            # 分析单个列的特征并推荐处理方法
            if unique_count < 10:
                col_choice = 'one_hot_encoder'  # 唯一值较少，使用独热编码
            elif unique_count < 100:
                col_choice = 'label_encoder'  # 唯一值适中，使用标签编码
            elif col_avg_length > 20:
                col_choice = 'tfidf'  # 长文本，使用TF-IDF
            else:
                col_choice = 'frequency_encoder'  # 唯一值较多，使用频率编码
            
            # 构建该列的参数配置
            col_params = {
                'str_choice': col_choice,
                'str_cols': col,  # 只包含当前列
                'tfidf_min_df': 1,
                'tfidf_max_df': 1.0,
                'tfidf_max_features': None,
                'lda_max_df': 1.0,
                'lda_min_df': 1,
                'lda_max_features': 1000,
                'lda_n_components': 10,
                'lda_random_state': 42,
                'use_chinese_tokenizer': False,
                'feature_selection': False,
                'k_features': 100
            }
            
            # 根据该列的特征调整参数
            if col_choice in ['tfidf', 'count']:
                # 根据唯一值数量调整特征数
                if unique_count > 1000:
                    col_params['tfidf_max_features'] = 1000
                elif unique_count > 500:
                    col_params['tfidf_max_features'] = 500
                
                # 根据样本数量调整最小文档频率
                if sample_count > 1000:
                    col_params['tfidf_min_df'] = 0.01
                elif sample_count > 100:
                    col_params['tfidf_min_df'] = 0.05
                
                # 根据样本数量调整最大文档频率
                if sample_count > 5000:
                    col_params['tfidf_max_df'] = 0.95
                elif sample_count > 1000:
                    col_params['tfidf_max_df'] = 0.98
            
            if col_choice == 'lda':
                # 根据唯一值数量调整主题数
                if unique_count > 1000:
                    col_params['lda_n_components'] = 20
                elif unique_count > 500:
                    col_params['lda_n_components'] = 15
                elif unique_count < 100:
                    col_params['lda_n_components'] = 5
                
                # 根据样本数量调整特征数
                if sample_count > 1000:
                    col_params['lda_max_features'] = 2000
                elif sample_count > 100:
                    col_params['lda_max_features'] = 1500
                
                # 根据样本数量调整最小文档频率
                if sample_count > 1000:
                    col_params['lda_min_df'] = 0.01
                elif sample_count > 100:
                    col_params['lda_min_df'] = 0.05
                
                # 根据样本数量调整最大文档频率
                if sample_count > 5000:
                    col_params['lda_max_df'] = 0.95
                elif sample_count > 1000:
                    col_params['lda_max_df'] = 0.98
                
                # 根据主题数调整随机种子
                col_params['lda_random_state'] = 42 + (col_params['lda_n_components'] % 10)
            
            # 自动检测是否需要中文分词
            has_chinese = False
            sample = col_data.sample(min(10, sample_count))
            for text in sample:
                # 确保text是字符串类型
                if isinstance(text, str):
                    if any('\u4e00' <= c <= '\u9fff' for c in text):
                        has_chinese = True
                        break
            
            if has_chinese:
                col_params['use_chinese_tokenizer'] = True
            
            # 根据文本长度决定是否进行特征选择
            if col_avg_length > 30:
                col_params['feature_selection'] = True
                # 根据唯一值数量和文本长度调整特征数量
                if unique_count > 500:
                    col_params['k_features'] = 300
                elif unique_count > 200:
                    col_params['k_features'] = 200
                else:
                    col_params['k_features'] = unique_count
            elif col_choice in ['tfidf', 'count', 'lda'] and unique_count > 500:
                # 对于高维特征，即使文本长度较短也考虑特征选择
                col_params['feature_selection'] = True
                col_params['k_features'] = 150 if unique_count > 300 else unique_count
            
            # 添加该列的分析信息
            analysis_info = {
                'unique_count': unique_count,
                'avg_length': col_avg_length,
                'sample_count': sample_count
            }
            
            # 为了显示分析信息，我们临时存储分析结果
            # 但在添加到参数列表前，我们创建一个不包含分析信息的副本
            col_params_copy = col_params.copy()
            col_params_copy['column_analysis'] = analysis_info
            column_params_list.append(col_params_copy)
    
    return column_params_list, string_columns

def get_string_columns(data):
    """获取数据中的字符串列
    
    Args:
        data: pandas DataFrame，输入数据
    
    Returns:
        list: 字符串列列表
    """
    string_columns = []
    for col in data.columns:
        # 跳过数值列
        if pd.api.types.is_numeric_dtype(data[col]):
            continue
        # 检查是否是字符串列
        if data[col].dtype == 'object' or data[col].dtype == 'string' or str(data[col].dtype) == 'str':
            # 检查是否是纯字符串列（非数值）
            try:
                pd.to_numeric(data[col])
            except ValueError:
                string_columns.append(col)
    return string_columns

def str_py(data, auto_test=False):
    """Python实现的字符串处理分析
    
    Args:
        data: pandas DataFrame，输入数据
        auto_test: bool，是否自动测试模式，默认为 False
    
    Returns:
        dict: 包含字符串处理参数的字典
    """
    # 首先获取字符串列
    string_columns = get_string_columns(data)
    
    if not string_columns:
        ColorPrinter.print_info("没有发现需要处理的字符串列")
        return 'skip'
    
    # 显示发现的字符串列信息
    ColorPrinter.print_info(f"发现 {len(string_columns)} 个字符串列需要处理")
    
    # 智能推荐菜单
    if auto_test:
        print("\033[31m下面进行测试使用智能推荐\033[0m")
        choice = "使用推荐方案"
    else:
        menu_options = [
            "使用推荐方案",
            "自定义处理方案",
            "跳过处理"
        ]
        menu = interactive(menu_options, title="字符串处理", description="请选择处理方式:(m键表示选择回车表示确认)" , min_select=1, max_select=1)
        
        selected = menu.run()
        selected_list = ensure_list(selected)
        choice = selected_list[0] if selected_list else "跳过处理"
    
    if choice == "跳过处理":
        ColorPrinter.print_info("跳过字符串处理")
        return 'skip'
    
    elif choice == "使用推荐方案":
        ColorPrinter.print_success("使用推荐方案")
        # 分析字符串分布并获取推荐参数
        column_params_list, _ = analyze_string_distribution(data)
        
        # 显示每个列的推荐处理方法
        ColorPrinter.print_info("每个字符串列的推荐处理方法:")
        for i, col_params in enumerate(column_params_list):
            col = col_params['str_cols']
            analysis = col_params['column_analysis']
            ColorPrinter.print_info(f"  {col}: {col_params['str_choice']} (唯一值: {analysis['unique_count']}, 平均长度: {analysis['avg_length']:.2f})")
        
        # 显示详细分析信息
        ColorPrinter.print_info("\n详细分析信息:")
        for i, col_params in enumerate(column_params_list):
            col = col_params['str_cols']
            analysis = col_params['column_analysis']
            ColorPrinter.print_info(f"  列 {col}:")
            ColorPrinter.print_info(f"    唯一值数量: {analysis['unique_count']} (不同值的数量，影响编码方法选择)")
            ColorPrinter.print_info(f"    平均长度: {analysis['avg_length']:.2f} (字符串平均长度，影响特征提取方法)")
            ColorPrinter.print_info(f"    样本数量: {analysis['sample_count']} (数据量大小，影响参数调优)")
            ColorPrinter.print_info(f"    中文分词: {col_params['use_chinese_tokenizer']} (是否使用中文分词器)")
            ColorPrinter.print_info(f"    特征选择: {col_params['feature_selection']} (是否进行特征选择)")
            if col_params['feature_selection']:
                ColorPrinter.print_info(f"    特征数量: {col_params['k_features']} (保留的特征数量)")
        
        # 构建结果，使用单个 str_preprocessing 字典包含所有列的处理参数
        result = {}
        str_preprocessing_params = {
            'str_ways': [],
            'tfidf_min_df': 1,
            'tfidf_max_df': 1.0,
            'tfidf_max_features': None,
            'lda_min_df': 1,
            'lda_max_df': 1.0,
            'lda_max_features': 1000,
            'lda_n_components': 10,
            'lda_random_state': 42
        }
        
        # 为每个列生成处理参数
        for i, col_params in enumerate(column_params_list):
            # 将 str_choice 改为 method 格式
            clean_params = col_params.copy()
            if 'column_analysis' in clean_params:
                del clean_params['column_analysis']
            # 将 str_choice 包装为 method 列表
            if 'str_choice' in clean_params:
                clean_params['method'] = [clean_params['str_choice']]
                del clean_params['str_choice']
            # 将每个列的参数添加到 str_ways 列表中
            str_preprocessing_params['str_ways'].append(clean_params)
        
        result['str_preprocessing'] = str_preprocessing_params
        
        # 可视化处理路线
        visualize_route(result)
        
        return result
    elif choice == "自定义处理方案":
        # 自定义处理方案
        ColorPrinter.print_info("自定义处理方案")
        
        # 1. 选择要处理的列
        col_menu = interactive(string_columns, title="选择列", description="请选择要处理的字符串列:", min_select=1, max_select=len(string_columns))
        selected_cols = col_menu.run()
        if not selected_cols:
            ColorPrinter.print_warning("未选择任何列，跳过处理")
            return 'skip'
        
        # 为每个选择的列单独选择处理方法和参数
        result = {}
        for i, col in enumerate(selected_cols):
            ColorPrinter.print_info(f"\n处理列: {col}")
            
            # 2. 为当前列选择处理方法
            method_options = [
                "label_encoder",
                "one_hot_encoder",
                "frequency_encoder",
                "count_encoder",
                "target_encoder",
                "tfidf",
                "count",
                "lda"
            ]
            method_menu = interactive(method_options, title=f"选择{col}的处理方法", description="请选择字符串处理方法:" , min_select=1, max_select=1)
            
            selected = method_menu.run()
            selected_list = ensure_list(selected)
            str_choice = selected_list[0] if selected_list else 'skip'
            
            # 3. 根据选择的处理方法，设置相应的参数
            col_params = {
                'str_choice': str_choice,
                'str_cols': col,  # 只包含当前列
                'tfidf_min_df': 1,
                'tfidf_max_df': 1.0,
                'tfidf_max_features': None,
                'lda_max_df': 1.0,
                'lda_min_df': 1,
                'lda_max_features': 1000,
                'lda_n_components': 10,
                'lda_random_state': 42,
                'use_chinese_tokenizer': False,
                'feature_selection': False,
                'k_features': 100
            }
            
            # 针对不同的处理方法，添加特定参数
            if str_choice in ['tfidf', 'count', 'lda']:
                # 分词器选择
                tokenizer_options = ["False", "True"]
                tokenizer_menu = interactive(tokenizer_options, title="分词器选择", description="请选择分词器: False=默认分词器(英文), True=中文分词器(jieba)")
                selected = tokenizer_menu.run()
                selected_list = ensure_list(selected)
                use_chinese_tokenizer = selected_list[0] if selected_list else "False"
                col_params['use_chinese_tokenizer'] = use_chinese_tokenizer == "True"
                
                # 特征选择
                feature_options = ["False", "True"]
                feature_menu = interactive(feature_options, title="特征选择", description="是否进行特征选择: False=不选择, True=使用卡方检验")
                selected = feature_menu.run()
                selected_list = ensure_list(selected)
                feature_selection = selected_list[0] if selected_list else "False"
                col_params['feature_selection'] = feature_selection == "True"
                
                if feature_selection == "True":
                    # 特征数量
                    k_features = input("请输入要保留的特征数量 (默认100): ")
                    col_params['k_features'] = int(k_features) if k_features else 100
            
            if str_choice in ['tfidf', 'count']:
                # TF-IDF/Count参数
                tfidf_min_df = input("请输入最小文档频率 (默认1): ")
                col_params['tfidf_min_df'] = float(tfidf_min_df) if tfidf_min_df else 1
                
                tfidf_max_df = input("请输入最大文档频率 (默认1.0): ")
                col_params['tfidf_max_df'] = float(tfidf_max_df) if tfidf_max_df else 1.0
                
                tfidf_max_features = input("请输入最大特征数 (默认None): ")
                col_params['tfidf_max_features'] = int(tfidf_max_features) if tfidf_max_features else None
            
            if str_choice == 'lda':
                # LDA参数
                lda_min_df = input("请输入最小文档频率 (默认1): ")
                col_params['lda_min_df'] = float(lda_min_df) if lda_min_df else 1
                
                lda_max_df = input("请输入最大文档频率 (默认1.0): ")
                col_params['lda_max_df'] = float(lda_max_df) if lda_max_df else 1.0
                
                lda_max_features = input("请输入最大特征数 (默认1000): ")
                col_params['lda_max_features'] = int(lda_max_features) if lda_max_features else 1000
                
                lda_n_components = input("请输入主题数 (默认10): ")
                col_params['lda_n_components'] = int(lda_n_components) if lda_n_components else 10
                
                lda_random_state = input("请输入随机种子 (默认42): ")
                col_params['lda_random_state'] = int(lda_random_state) if lda_random_state else 42
            
        # 构建结果，使用单个 str_preprocessing 字典包含所有列的处理参数
        str_preprocessing_params = {
            'str_ways': [],
            'tfidf_min_df': 1,
            'tfidf_max_df': 1.0,
            'tfidf_max_features': None,
            'lda_min_df': 1,
            'lda_max_df': 1.0,
            'lda_max_features': 1000,
            'lda_n_components': 10,
            'lda_random_state': 42
        }
        
        # 将每个列的参数添加到 str_ways 列表中，并转换格式
        for col, col_params in result.items():
            # 将 str_choice 改为 method 格式
            if 'str_choice' in col_params:
                col_params['method'] = [col_params['str_choice']]
                del col_params['str_choice']
            str_preprocessing_params['str_ways'].append(col_params)
        
        final_result = {'str_preprocessing': str_preprocessing_params}
        
        # 可视化处理路线
        visualize_route(final_result)
        
        return final_result


if __name__ == "__main__":
    """
    测试字符串处理功能
    """
    import pandas as pd
    import numpy as np
    
    print("=" * 80)
    print("字符串处理模块测试")
    print("=" * 80)
    
    # 创建测试数据
    data = pd.DataFrame({
        'category': ['A', 'B', 'C', 'A', 'B', 'C', 'A', 'B', 'C', 'A'],
        'short_text': ['apple', 'banana', 'orange', 'apple', 'banana', 'orange', 'apple', 'banana', 'orange', 'apple'],
        'long_text': [
            'This is a long text about machine learning and data science',
            'Natural language processing is a fascinating field of study',
            'Python is a popular programming language for data analysis',
            'Machine learning algorithms can learn from data patterns',
            'Deep learning models are used for image recognition',
            'Data visualization helps in understanding complex patterns',
            'Statistical analysis is important for data-driven decisions',
            'Feature engineering is crucial for machine learning models',
            'Model evaluation metrics help assess performance',
            'Hyperparameter tuning improves model accuracy'
        ],
        'chinese_text': [
            '这是一段中文文本',
            '机器学习是一个有趣的领域',
            '数据分析需要专业知识',
            '人工智能正在改变世界',
            'Python是一种流行的编程语言',
            '数据科学是一个快速发展的领域',
            '深度学习模型在图像识别中表现出色',
            '自然语言处理技术不断进步',
            '数据可视化帮助理解复杂数据',
            '统计分析是数据驱动决策的基础'
        ],
        'numeric_column': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    })
    
    print("测试数据:")
    print(data)
    print(f"\n数据形状: {data.shape}")
    print(f"列名: {list(data.columns)}")
    
    print("\n" + "=" * 80)
    print("测试字符串列检测")
    print("=" * 80)
    
    # 测试字符串列检测
    string_cols = get_string_columns(data)
    print(f"检测到的字符串列: {string_cols}")
    
    print("\n" + "=" * 80)
    print("测试字符串分布分析")
    print("=" * 80)
    
    # 测试字符串分布分析
    column_params_list, _ = analyze_string_distribution(data)
    for i, col_params in enumerate(column_params_list):
        col = col_params['str_cols']
        analysis = col_params['column_analysis']
        print(f"\n列 {col}:")
        print(f"  推荐处理方法: {col_params['str_choice']}")
        print(f"  唯一值数量: {analysis['unique_count']}")
        print(f"  平均长度: {analysis['avg_length']:.2f}")
        print(f"  样本数量: {analysis['sample_count']}")
        print(f"  中文分词: {col_params['use_chinese_tokenizer']}")
        print(f"  特征选择: {col_params['feature_selection']}")
        if col_params['feature_selection']:
            print(f"  特征数量: {col_params['k_features']}")
    
    print("\n" + "=" * 80)
    print("测试主处理函数")
    print("=" * 80)
    
    # 测试主处理函数
    try:
        result = str_py(data)
        print("\n处理结果:")
        if result == 'skip':
            print("跳过处理")
        else:
            for key, value in result.items():
                print(f"\n{key}:")
                print(f"  处理方法: {value['str_choice']}")
                print(f"  处理列: {value['str_cols']}")
                print(f"  中文分词: {value['use_chinese_tokenizer']}")
                print(f"  特征选择: {value['feature_selection']}")
    except Exception as e:
        print(f"测试过程中出错: {e}")
    
    print("\n" + "=" * 80)
    print("测试完成!")
    print("=" * 80)
