#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
任务三: 数据分析模型构建与验证
构建产品销售量预测模型
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import warnings
warnings.filterwarnings('ignore')

# 设置中文显示
plt.rcParams['font.sans-serif'] = ['Arial Unicode MS', 'SimHei']
plt.rcParams['axes.unicode_minus'] = False

print("=" * 80)
print("跨境电商数据分析竞赛 - 任务三:模型构建与验证")
print("=" * 80)

# ============================================
# 3.3.1 加载数据
# ============================================
print("\n【步骤1】加载清洗后的数据")
print("-" * 80)

df = pd.read_csv('../data/ecommerce_sales_cleaned.csv', encoding='utf-8-sig')
df['order_date'] = pd.to_datetime(df['order_date'])
df['product_launch_date'] = pd.to_datetime(df['product_launch_date'])

print(f"✓ 数据加载完成: {len(df)} 条记录")
print(f"  字段数量: {len(df.columns)}")

# ============================================
# 3.3.2 数据预处理 - 特征工程
# ============================================
print("\n【步骤2】特征工程")
print("-" * 80)

# 创建特征副本
feature_df = df.copy()

# 2.1 时间特征
print("\n构建时间特征...")
feature_df['year'] = feature_df['order_date'].dt.year
feature_df['month'] = feature_df['order_date'].dt.month
feature_df['quarter'] = feature_df['order_date'].dt.quarter
feature_df['day_of_week'] = feature_df['order_date'].dt.dayofweek
feature_df['is_weekend'] = feature_df['day_of_week'].isin([5, 6]).astype(int)
feature_df['days_since_launch'] = (feature_df['order_date'] - feature_df['product_launch_date']).dt.days

print(f"  ✓ 时间特征: year, month, quarter, day_of_week, is_weekend, days_since_launch")

# 2.2 价格特征
print("\n构建价格特征...")
feature_df['price_range'] = pd.cut(feature_df['unit_price'], 
                                   bins=[0, 50, 150, 300, float('inf')],
                                   labels=['低价', '中低价', '中高价', '高价'])
feature_df['discount_level'] = pd.cut(feature_df['discount_rate'],
                                      bins=[-0.01, 0, 0.1, 0.2, 1],
                                      labels=['无折扣', '小折扣', '中折扣', '大折扣'])

# 成本利润率
feature_df['cost_margin'] = (feature_df['unit_price'] - feature_df['cost_price']) / feature_df['unit_price']

print(f"  ✓ 价格特征: price_range, discount_level, cost_margin")

# 2.3 统计特征 - 产品历史表现
print("\n构建统计特征...")

# 按产品计算历史销售统计
product_stats = df.groupby('product_id').agg({
    'quantity': ['mean', 'std', 'sum'],
    'total_amount': ['mean', 'sum'],
    'order_id': 'count'
}).reset_index()

product_stats.columns = ['product_id', 'product_avg_qty', 'product_std_qty', 'product_total_qty',
                        'product_avg_amount', 'product_total_amount', 'product_order_count']

feature_df = feature_df.merge(product_stats, on='product_id', how='left')

# 按类别计算统计
category_stats = df.groupby('category').agg({
    'quantity': 'mean',
    'total_amount': 'mean'
}).reset_index()

category_stats.columns = ['category', 'category_avg_qty', 'category_avg_amount']
feature_df = feature_df.merge(category_stats, on='category', how='left')

print(f"  ✓ 统计特征: product历史均值/标准差/总量, category平均值")

# 2.4 类别编码
print("\n对类别变量进行编码...")

# Label Encoding for tree-based models
le_encoders = {}
categorical_cols = ['category', 'sub_category', 'region', 'country', 'price_range', 'discount_level']

for col in categorical_cols:
    le = LabelEncoder()
    feature_df[f'{col}_encoded'] = le.fit_transform(feature_df[col].astype(str))
    le_encoders[col] = le

print(f"  ✓ Label编码完成: {len(categorical_cols)} 个类别字段")

# 2.5 交叉特征
print("\n构建交叉特征...")
feature_df['region_category'] = feature_df['region'] + '_' + feature_df['category']
le_region_category = LabelEncoder()
feature_df['region_category_encoded'] = le_region_category.fit_transform(feature_df['region_category'])

print(f"  ✓ 交叉特征: region_category")

# ============================================
# 3.3.2 特征选择
# ============================================
print("\n【步骤3】特征选择")
print("-" * 80)

# 定义特征列(排除目标变量和无关字段)
feature_columns = [
    # 时间特征
    'year', 'month', 'quarter', 'day_of_week', 'is_weekend', 'days_since_launch',
    # 价格特征
    'unit_price', 'discount_rate', 'cost_price', 'cost_margin',
    # 库存特征
    'stock_quantity',
    # 物流特征
    'shipping_cost', 'delivery_days',
    # 编码特征
    'category_encoded', 'sub_category_encoded', 'region_encoded', 
    'country_encoded', 'price_range_encoded', 'discount_level_encoded',
    'region_category_encoded',
    # 统计特征
    'product_avg_qty', 'product_std_qty', 'product_order_count',
    'category_avg_qty'
]

# 目标变量
target = 'quantity'

# 准备训练数据
X = feature_df[feature_columns].copy()
y = feature_df[target].copy()

# 处理缺失值
X = X.fillna(X.median())

print(f"✓ 特征数量: {len(feature_columns)}")
print(f"✓ 样本数量: {len(X)}")
print(f"✓ 目标变量: {target}")

# ============================================
# 3.3.2 数据标准化
# ============================================
print("\n【步骤4】数据标准化")
print("-" * 80)

# 分离训练集和测试集 (按时间切分)
# 使用前80%作为训练,后20%作为测试
split_index = int(len(X) * 0.8)
X_train, X_test = X[:split_index], X[split_index:]
y_train, y_test = y[:split_index], y[split_index:]

print(f"✓ 训练集大小: {len(X_train)} ({len(X_train)/len(X)*100:.1f}%)")
print(f"✓ 测试集大小: {len(X_test)} ({len(X_test)/len(X)*100:.1f}%)")

# 标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

print(f"✓ 特征标准化完成")

# ============================================
# 3.3.3 模型训练
# ============================================
print("\n【步骤5】模型训练")
print("-" * 80)

models = {
    'Linear Regression': LinearRegression(),
    'Ridge Regression': Ridge(alpha=1.0),
    'Lasso Regression': Lasso(alpha=0.1),
    'Decision Tree': DecisionTreeRegressor(max_depth=10, random_state=42),
    'Random Forest': RandomForestRegressor(n_estimators=100, max_depth=10, random_state=42, n_jobs=-1),
    'Gradient Boosting': GradientBoostingRegressor(n_estimators=100, max_depth=5, random_state=42)
}

results = []

for name, model in models.items():
    print(f"\n训练 {name}...")
    
    # 选择是否使用标准化数据
    if 'Linear' in name or 'Ridge' in name or 'Lasso' in name:
        model.fit(X_train_scaled, y_train)
        y_train_pred = model.predict(X_train_scaled)
        y_test_pred = model.predict(X_test_scaled)
    else:
        model.fit(X_train, y_train)
        y_train_pred = model.predict(X_train)
        y_test_pred = model.predict(X_test)
    
    # 计算评估指标
    train_mse = mean_squared_error(y_train, y_train_pred)
    train_rmse = np.sqrt(train_mse)
    train_mae = mean_absolute_error(y_train, y_train_pred)
    train_r2 = r2_score(y_train, y_train_pred)
    
    test_mse = mean_squared_error(y_test, y_test_pred)
    test_rmse = np.sqrt(test_mse)
    test_mae = mean_absolute_error(y_test, y_test_pred)
    test_r2 = r2_score(y_test, y_test_pred)
    
    # MAPE (过滤掉y_test为0的情况)
    mask = y_test > 0
    test_mape = np.mean(np.abs((y_test[mask] - y_test_pred[mask]) / y_test[mask])) * 100
    
    results.append({
        'Model': name,
        'Train_RMSE': train_rmse,
        'Train_MAE': train_mae,
        'Train_R2': train_r2,
        'Test_RMSE': test_rmse,
        'Test_MAE': test_mae,
        'Test_R2': test_r2,
        'Test_MAPE': test_mape,
        'model_object': model
    })
    
    print(f"  训练集 - RMSE: {train_rmse:.4f}, MAE: {train_mae:.4f}, R²: {train_r2:.4f}")
    print(f"  测试集 - RMSE: {test_rmse:.4f}, MAE: {test_mae:.4f}, R²: {test_r2:.4f}, MAPE: {test_mape:.2f}%")

# ============================================
# 3.3.4 模型评估对比
# ============================================
print("\n" + "=" * 80)
print("【步骤6】模型评估对比")
print("-" * 80)

results_df = pd.DataFrame(results)
results_df = results_df.sort_values('Test_R2', ascending=False)

print("\n模型性能对比表:")
print(results_df[['Model', 'Train_R2', 'Test_R2', 'Test_RMSE', 'Test_MAE', 'Test_MAPE']].to_string(index=False))

# 选择最佳模型
best_model_name = results_df.iloc[0]['Model']
best_model = results_df.iloc[0]['model_object']
print(f"\n✓ 最佳模型: {best_model_name}")
print(f"  测试集R²: {results_df.iloc[0]['Test_R2']:.4f}")
print(f"  测试集RMSE: {results_df.iloc[0]['Test_RMSE']:.4f}")

# ============================================
# 3.3.5 交叉验证
# ============================================
print("\n【步骤7】5折交叉验证")
print("-" * 80)

# 使用最佳模型进行交叉验证
if 'Linear' in best_model_name or 'Ridge' in best_model_name or 'Lasso' in best_model_name:
    X_for_cv = X_train_scaled
else:
    X_for_cv = X_train

cv_scores = cross_val_score(best_model, X_for_cv, y_train, cv=5, 
                            scoring='r2', n_jobs=-1)

print(f"✓ 交叉验证R²分数: {cv_scores}")
print(f"  平均R²: {cv_scores.mean():.4f}")
print(f"  标准差: {cv_scores.std():.4f}")
print(f"\n模型稳定性评估: ", end='')
if cv_scores.std() < 0.05:
    print("优秀 (标准差<0.05)")
elif cv_scores.std() < 0.1:
    print("良好 (标准差<0.1)")
else:
    print("一般 (标准差>=0.1)")

# ============================================
# 3.3.6 特征重要性分析
# ============================================
print("\n【步骤8】特征重要性分析")
print("-" * 80)

if hasattr(best_model, 'feature_importances_'):
    # 树模型的特征重要性
    feature_importance = pd.DataFrame({
        'Feature': feature_columns,
        'Importance': best_model.feature_importances_
    }).sort_values('Importance', ascending=False)
    
    print(f"\nTop 15 重要特征:")
    print(feature_importance.head(15).to_string(index=False))
    
    # 可视化特征重要性
    plt.figure(figsize=(12, 8))
    top_features = feature_importance.head(15)
    plt.barh(range(len(top_features)), top_features['Importance'])
    plt.yticks(range(len(top_features)), top_features['Feature'])
    plt.xlabel('Feature Importance')
    plt.title(f'{best_model_name} - Top 15 Feature Importance')
    plt.tight_layout()
    plt.savefig('../visualization/feature_importance.png', dpi=300, bbox_inches='tight')
    print(f"\n✓ 特征重要性图表已保存: visualization/feature_importance.png")
    
elif hasattr(best_model, 'coef_'):
    # 线性模型的系数
    feature_importance = pd.DataFrame({
        'Feature': feature_columns,
        'Coefficient': best_model.coef_
    }).sort_values('Coefficient', key=abs, ascending=False)
    
    print(f"\nTop 15 影响特征:")
    print(feature_importance.head(15).to_string(index=False))

# ============================================
# 业务洞察
# ============================================
print("\n" + "=" * 80)
print("【业务洞察】")
print("-" * 80)

print("\n关键影响因素:")
print("1. 历史销售表现: 产品历史销售量和订单数对未来销售有显著影响")
print("2. 时间因素: 月份、季度等时间特征反映了季节性需求")
print("3. 定价策略: 价格水平和折扣力度影响销量")
print("4. 产品生命周期: 上架天数反映产品成熟度")
print("5. 区域和品类: 不同区域和品类有不同的销售特征")

print("\n模型应用建议:")
print("1. 库存预测: 基于模型预测结果优化各产品的库存水平")
print("2. 选品决策: 识别高潜力产品特征,指导新品选择")
print("3. 定价优化: 分析价格对销量的影响,制定最优定价策略")
print("4. 促销计划: 预测促销活动对销量的提升效果")

# ============================================
# 保存模型和结果
# ============================================
import pickle

model_file = '../data/sales_prediction_model.pkl'
with open(model_file, 'wb') as f:
    pickle.dump({
        'model': best_model,
        'scaler': scaler,
        'feature_columns': feature_columns,
        'model_name': best_model_name
    }, f)

print(f"\n✓ 模型已保存至: {model_file}")

# 保存预测结果
prediction_results = pd.DataFrame({
    'Actual': y_test,
    'Predicted': y_test_pred if 'Linear' in best_model_name or 'Ridge' in best_model_name or 'Lasso' in best_model_name 
                 else best_model.predict(X_test)
})
prediction_results.to_csv('../data/prediction_results.csv', index=False)
print(f"✓ 预测结果已保存至: data/prediction_results.csv")

print("\n" + "=" * 80)
print("模型构建与验证任务完成!")
print("=" * 80)
