#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
任务一: 数据清洗参考脚本
包含缺失值、异常值、重复值处理,数据类型转换和数据验证
"""

import pandas as pd
import numpy as np
from datetime import datetime
import warnings
warnings.filterwarnings('ignore')

print("=" * 80)
print("跨境电商数据分析竞赛 - 任务一:数据清洗")
print("=" * 80)

# ============================================
# 3.1.1 数据读取与探索
# ============================================
print("\n【步骤1】数据读取与探索")
print("-" * 80)

# 读取原始数据
df = pd.read_csv('../data/ecommerce_sales_raw.csv', encoding='utf-8-sig')
print(f"✓ 成功读取数据集")
print(f"  数据维度: {df.shape[0]} 行 × {df.shape[1]} 列")
print(f"  内存占用: {df.memory_usage(deep=True).sum() / 1024 / 1024:.2f} MB")

# 查看基本信息
print(f"\n数据字段信息:")
print(df.info())

# 统计特征
print(f"\n数值字段统计描述:")
print(df.describe())

# 保存数据质量报告
quality_report = []

# ============================================
# 3.1.2 缺失值处理
# ============================================
print("\n" + "=" * 80)
print("【步骤2】缺失值处理")
print("-" * 80)

# 识别并统计缺失情况
missing_stats = df.isnull().sum()
missing_pct = (df.isnull().sum() / len(df) * 100).round(2)
missing_df = pd.DataFrame({
    '缺失数量': missing_stats,
    '缺失比例(%)': missing_pct
})
missing_df = missing_df[missing_df['缺失数量'] > 0].sort_values('缺失数量', ascending=False)

print(f"缺失值统计:")
print(missing_df)
quality_report.append(f"\n原始数据缺失值情况:\n{missing_df}")

# 处理sub_category缺失值
print(f"\n处理 sub_category 缺失值...")
before_missing = df['sub_category'].isnull().sum()

# 方法:根据product_name推断,如果无法推断则根据category的众数填充
def infer_subcategory(row):
    if pd.isna(row['sub_category']):
        # 尝试从相同product_id的其他记录中获取
        same_product = df[df['product_id'] == row['product_id']]['sub_category'].dropna()
        if len(same_product) > 0:
            return same_product.mode()[0]
        # 否则使用该类别的众数
        category_mode = df[df['category'] == row['category']]['sub_category'].mode()
        if len(category_mode) > 0:
            return category_mode[0]
    return row['sub_category']

df['sub_category'] = df.apply(infer_subcategory, axis=1)
after_missing = df['sub_category'].isnull().sum()
print(f"  处理前缺失: {before_missing}, 处理后缺失: {after_missing}")
print(f"  业务合理性: 根据产品ID或类别众数推断,保持产品分类的一致性")

# 处理cost_price缺失值
print(f"\n处理 cost_price 缺失值...")
before_missing = df['cost_price'].isnull().sum()

# 方法:使用同类别产品的成本价中位数填充
df['cost_price'] = df.groupby('category')['cost_price'].transform(
    lambda x: x.fillna(x.median())
)

# 如果还有缺失,使用unit_price * 0.45估算
df['cost_price'] = df.apply(
    lambda row: row['unit_price'] * 0.45 if pd.isna(row['cost_price']) else row['cost_price'],
    axis=1
)

after_missing = df['cost_price'].isnull().sum()
print(f"  处理前缺失: {before_missing}, 处理后缺失: {after_missing}")
print(f"  业务合理性: 使用同类别中位数或售价45%估算,符合成本定价逻辑")

# 处理discount_rate缺失值
print(f"\n处理 discount_rate 缺失值...")
before_missing = df['discount_rate'].isnull().sum()

# 方法:缺失视为无折扣,填充0
df['discount_rate'].fillna(0, inplace=True)

after_missing = df['discount_rate'].isnull().sum()
print(f"  处理前缺失: {before_missing}, 处理后缺失: {after_missing}")
print(f"  业务合理性: 缺失折扣率视为正价销售,填充0符合业务逻辑")

# 处理shipping_cost缺失值
print(f"\n处理 shipping_cost 缺失值...")
before_missing = df['shipping_cost'].isnull().sum()

# 方法:根据区域和产品类别的平均运费填充
df['shipping_cost'] = df.groupby(['region', 'category'])['shipping_cost'].transform(
    lambda x: x.fillna(x.mean())
)

# 如果还有缺失,使用区域平均
df['shipping_cost'] = df.groupby('region')['shipping_cost'].transform(
    lambda x: x.fillna(x.mean())
)

after_missing = df['shipping_cost'].isnull().sum()
print(f"  处理前缺失: {before_missing}, 处理后缺失: {after_missing}")
print(f"  业务合理性: 按区域和品类平均运费填充,符合物流成本特征")

# 处理supplier_id缺失值
print(f"\n处理 supplier_id 缺失值...")
before_missing = df['supplier_id'].isnull().sum()

# 方法:标记为未知供应商
df['supplier_id'].fillna('S999', inplace=True)

after_missing = df['supplier_id'].isnull().sum()
print(f"  处理前缺失: {before_missing}, 处理后缺失: {after_missing}")
print(f"  业务合理性: 标记为未知供应商S999,便于后续分析")

# ============================================
# 3.1.3 异常值处理
# ============================================
print("\n" + "=" * 80)
print("【步骤3】异常值处理")
print("-" * 80)

# 处理quantity异常值
print(f"\n处理 quantity 异常值...")
abnormal_qty = ((df['quantity'] < 0) | (df['quantity'] > 10000)).sum()
print(f"  异常值数量: {abnormal_qty}")

# 删除负数和超大值
before_count = len(df)
df = df[(df['quantity'] > 0) & (df['quantity'] <= 100)]
after_count = len(df)
print(f"  删除记录: {before_count - after_count} 条")
print(f"  业务合理性: 单笔订单超过100件或负数不符合业务逻辑,予以删除")

# 处理unit_price异常值
print(f"\n处理 unit_price 异常值...")
# 先转换为数值类型
df['unit_price'] = pd.to_numeric(df['unit_price'], errors='coerce')

abnormal_price = ((df['unit_price'] <= 0) | (df['unit_price'] > 5000)).sum()
print(f"  异常值数量: {abnormal_price}")

# 删除0价格和超高价格
before_count = len(df)
df = df[(df['unit_price'] > 0) & (df['unit_price'] <= 5000)]
after_count = len(df)
print(f"  删除记录: {before_count - after_count} 条")
print(f"  业务合理性: 单价超过5000美元或为0不符合产品定价逻辑")

# 处理delivery_days异常值
print(f"\n处理 delivery_days 异常值...")
abnormal_delivery = ((df['delivery_days'] < 0) | (df['delivery_days'] > 60)).sum()
print(f"  异常值数量: {abnormal_delivery}")

# 修正为区域平均配送时间
region_avg_delivery = df.groupby('region')['delivery_days'].median().to_dict()
df['delivery_days'] = df.apply(
    lambda row: region_avg_delivery[row['region']] 
    if (row['delivery_days'] < 0 or row['delivery_days'] > 60) 
    else row['delivery_days'],
    axis=1
)
print(f"  修正方法: 使用区域中位数配送时间替换异常值")
print(f"  业务合理性: 配送时间超过60天或负数不合理,用区域平均替换")

# 处理discount_rate异常值
print(f"\n处理 discount_rate 异常值...")
df['discount_rate'] = pd.to_numeric(df['discount_rate'], errors='coerce')
df['discount_rate'].fillna(0, inplace=True)

abnormal_discount = ((df['discount_rate'] < 0) | (df['discount_rate'] > 1)).sum()
print(f"  异常值数量: {abnormal_discount}")

# 修正为合理范围
df.loc[df['discount_rate'] < 0, 'discount_rate'] = 0
df.loc[df['discount_rate'] > 1, 'discount_rate'] = 0.3  # 假设为最大促销力度
print(f"  修正方法: 负值改为0,超过1的改为0.3")
print(f"  业务合理性: 折扣率应在0-1之间,超出范围进行修正")

# ============================================
# 3.1.4 重复值处理
# ============================================
print("\n" + "=" * 80)
print("【步骤4】重复值处理")
print("-" * 80)

# 识别完全重复的订单
print(f"\n处理完全重复订单...")
before_count = len(df)
duplicates = df.duplicated(subset=['order_id']).sum()
print(f"  完全重复订单数: {duplicates}")

# 保留第一条,删除重复
df.drop_duplicates(subset=['order_id'], keep='first', inplace=True)
after_count = len(df)
print(f"  删除记录: {before_count - after_count} 条")

# 处理产品名称不一致
print(f"\n标准化产品名称...")
# 统一转换为大写,去除多余空格
df['product_name'] = df['product_name'].str.strip().str.replace('-', '_')
print(f"  标准化方法: 统一分隔符为下划线,去除空格")
print(f"  业务合理性: 保持产品命名一致性,便于后续分析")

# ============================================
# 3.1.5 数据类型转换
# ============================================
print("\n" + "=" * 80)
print("【步骤5】数据类型转换")
print("-" * 80)

# 转换order_date
print(f"\n转换 order_date 为标准日期格式...")
def parse_date(date_str):
    """尝试多种日期格式解析"""
    if pd.isna(date_str):
        return None
    
    formats = ['%Y-%m-%d', '%m/%d/%Y', '%d-%m-%Y']
    for fmt in formats:
        try:
            return pd.to_datetime(date_str, format=fmt)
        except:
            continue
    return None

df['order_date'] = df['order_date'].apply(parse_date)
print(f"  转换成功: {df['order_date'].notna().sum()} 条")
print(f"  转换失败: {df['order_date'].isna().sum()} 条")

# 删除日期转换失败的记录
df = df[df['order_date'].notna()]

# 转换product_launch_date
print(f"\n转换 product_launch_date 为标准日期格式...")
df['product_launch_date'] = df['product_launch_date'].apply(parse_date)
df = df[df['product_launch_date'].notna()]

# 转换is_returned
print(f"\n转换 is_returned 为布尔类型...")
def parse_boolean(val):
    """统一转换为布尔值"""
    if pd.isna(val):
        return False
    val_str = str(val).strip().upper()
    return val_str in ['TRUE', '1', 'Y', '是']

df['is_returned'] = df['is_returned'].apply(parse_boolean)
print(f"  转换完成,退货率: {df['is_returned'].sum() / len(df) * 100:.2f}%")

# 转换数值字段
print(f"\n转换数值字段...")
numeric_cols = ['quantity', 'unit_price', 'total_amount', 'cost_price', 
                'discount_rate', 'shipping_cost', 'delivery_days', 'stock_quantity']

for col in numeric_cols:
    df[col] = pd.to_numeric(df[col], errors='coerce')

# 删除关键数值字段缺失的记录
df = df.dropna(subset=['quantity', 'unit_price', 'total_amount'])

print(f"  数值字段转换完成")
print(f"  当前数据集: {len(df)} 条")

# ============================================
# 3.1.6 数据验证
# ============================================
print("\n" + "=" * 80)
print("【步骤6】数据验证")
print("-" * 80)

# 验证total_amount计算一致性
print(f"\n验证 total_amount 计算一致性...")
df['calculated_amount'] = df['quantity'] * df['unit_price'] * (1 - df['discount_rate'])
df['calculated_amount'] = df['calculated_amount'].round(2)

# 计算差异(允许0.5元误差)
inconsistent = (abs(df['total_amount'] - df['calculated_amount']) > 0.5).sum()
print(f"  计算不一致记录: {inconsistent} 条")

# 修正不一致的金额
df.loc[abs(df['total_amount'] - df['calculated_amount']) > 0.5, 'total_amount'] = \
    df.loc[abs(df['total_amount'] - df['calculated_amount']) > 0.5, 'calculated_amount']

print(f"  已修正为计算值")

# 删除辅助列
df.drop('calculated_amount', axis=1, inplace=True)

# 验证order_date与product_launch_date逻辑关系
print(f"\n验证订单日期与上架日期的逻辑关系...")
date_errors = (df['order_date'] < df['product_launch_date']).sum()
print(f"  订单日期早于上架日期: {date_errors} 条")

# 删除逻辑错误的记录
df = df[df['order_date'] >= df['product_launch_date']]
print(f"  已删除逻辑错误记录")

# ============================================
# 生成清洗后的数据质量报告
# ============================================
print("\n" + "=" * 80)
print("【数据清洗质量报告】")
print("-" * 80)

print(f"\n✓ 清洗完成!")
print(f"  最终数据集: {len(df)} 行 × {len(df.columns)} 列")
print(f"  数据完整性: {(1 - df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100:.2f}%")

print(f"\n字段数据类型:")
print(df.dtypes)

print(f"\n数值字段统计:")
print(df[numeric_cols].describe())

print(f"\n类别字段分布:")
for col in ['category', 'region', 'country']:
    print(f"\n{col}:")
    print(df[col].value_counts())

# 保存清洗后的数据
output_file = '../data/ecommerce_sales_cleaned.csv'
df.to_csv(output_file, index=False, encoding='utf-8-sig')
print(f"\n✓ 清洗后数据已保存至: {output_file}")

# 生成质量报告文件
report_file = '../data/data_quality_report.txt'
with open(report_file, 'w', encoding='utf-8') as f:
    f.write("=" * 80 + "\n")
    f.write("数据清洗质量报告\n")
    f.write("=" * 80 + "\n\n")
    f.write(f"清洗时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
    f.write(f"原始数据: 30600 条记录\n")
    f.write(f"清洗后数据: {len(df)} 条记录\n")
    f.write(f"数据保留率: {len(df) / 30600 * 100:.2f}%\n\n")
    f.write("清洗步骤摘要:\n")
    f.write("1. 缺失值处理: 5个字段,采用业务逻辑填充\n")
    f.write("2. 异常值处理: 4个字段,删除或修正异常记录\n")
    f.write("3. 重复值处理: 去除完全重复订单,标准化产品名称\n")
    f.write("4. 数据类型转换: 日期、布尔值、数值字段统一格式\n")
    f.write("5. 数据验证: 修正金额计算错误,删除日期逻辑错误\n\n")
    f.write(f"数据完整性: {(1 - df.isnull().sum().sum() / (len(df) * len(df.columns))) * 100:.2f}%\n")

print(f"✓ 质量报告已保存至: {report_file}")

print("\n" + "=" * 80)
print("数据清洗任务完成!")
print("=" * 80)
