import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
import pymysql
from pyecharts.charts import Bar, Line, Pie

# 创建电商模拟数据
data = {
    'order_id': range(1, 1001),
    'user_id': np.random.randint(1001, 1100, 1000),
    'product_id': np.random.randint(2001, 2050, 1000),
    'category': np.random.choice(['电子产品', '服装', '家居', '美妆', '食品'], 1000),
    'price': np.random.normal(150, 50, 1000),
    'quantity': np.random.randint(1, 10, 1000),
    'order_date': pd.date_range('2024-01-01', periods=1000, freq='D'),
    'user_age': np.random.randint(18, 65, 1000),
    'user_income': np.random.normal(50000, 15000, 1000),
    'browse_time': np.random.normal(300, 100, 1000)
}
df = pd.DataFrame(data)
df['total_amount'] = df['quantity'] * df['price']
df.iloc[10:15, 4] = np.nan  #
df.iloc[20:25, 5] = np.nan
df.iloc[30:35, 7] = np.nan
df.iloc[100, 4] = -100
df.iloc[100, 5] = 100
df = pd.concat([df, df.iloc[50:55]])

print("原始数据形状：", df.shape)
print("前五行：", df.head(5))
print("数据基本信息：\n")
print(df.info())


# 数据清洗
# 处理缺失值
def deal_missing_value(df):
    # 数值取中位数
    numeric_cols = ['price', 'quantity', 'total_amount', 'user_income', 'browse_time', 'user_age']
    for col in numeric_cols:
        df[col] = df[col].fillna(df[col].median())
    # 分类取众数
    category_cols = ['category']
    for col in category_cols:
        df[col] = df[col].fillna(df[col].mode())
    return df

# 处理异常值
def deal_error_value(df):
    # 价格异常处理
    # 价格异常处理（移除负值和大于1000的值）
    print(f"deal_error_value 开始  {len(df)}")
    df = df[(df['price'] > 0) & (df['price'] < 1000)]
    # 数量异常处理（移除大于20的值）
    df = df[df['quantity'] <= 20]
    # 使用IQR方法处理用户收入异常值
    Q1 = df['user_income'].quantile(0.25)
    Q3 = df['user_income'].quantile(0.75)
    IQR = Q3 - Q1
    df = df[(df['user_income'] >= Q1 - IQR * 1.5) & (df['user_income'] <= Q3 + IQR * 1.5)]
    print(f"deal_error_value 结束  {len(df)}")
    return df


# 处理重复值
def deal_duplicates_value(df):
    # 基于订单ID去重
    df = df.drop_duplicates(subset=['order_id'], keep='first')
    return df


# 处理类型转换
def deal_type_change(df):
    df['order_id'] = df['order_id'].astype('str')
    df['user_id'] = df['user_id'].astype('str')
    df['order_date'] = pd.to_datetime(df['order_date'])
    return df


# 执行数据清洗
df_clean = df.copy()
print(df_clean.describe())
df_clean = deal_missing_value(df_clean)
df_clean = deal_error_value(df_clean)
df_clean = deal_duplicates_value(df_clean)
df_clean = deal_type_change(df_clean)

print("清洗后数据形状:", df_clean.shape)
print("\n清洗后数据描述性统计:")
print(df_clean.describe())
print(df_clean.head())

# 添加衍生特征
df_clean['month'] = df_clean['order_date'].dt.month
df_clean['day_of_week'] = df_clean['order_date'].dt.dayofweek
df_clean['is_weekend'] = df_clean['day_of_week'].isin([6, 7])
df_clean.to_csv('./clean.csv')
