#!/usr/bin/env python
# -*- coding: utf-8 -*-

# 说明：
#    装对Phone文件做去重
# History:
# Date          Author    Version       Modification
# --------------------------------------------------------------------------------------------------
# 2026/2/2    xiatn     V00.01.000    新建
# --------------------------------------------------------------------------------------------------
import os
import fnmatch
import hashlib
from tqdm import tqdm
from xtn_tools_pro.utils.log import Log
from xtn_tools_pro.utils.helpers import get_orderId_random
from xtn_tools_pro.utils.file_utils import get_file_line_cnt, get_filename, mkdirs_dir, get_file_extension, is_dir, \
    get_listdir


class FileDeduper:
    def __init__(self):
        self.__logger = Log('set_data', './xxx.log', log_level='DEBUG', is_write_to_console=True,
                            is_write_to_file=False,
                            color=True, mode='a', save_time_log_path='./logs')

    def dedup_file(self, inp_file, out_dir, out_file_name=None):
        """
            对单文件去重
        :param inp_file: 需要去重的文件
        :param out_dir:去重后文件保存在哪个目录下,如果目录不存在则会自动创建
        :param out_file_name:去重后文件名称,如果不传 默认使用的是 <inp_file> 的文件名
        :return:
        """
        if get_file_extension(inp_file) != ".txt":
            self.__logger.critical("文件不合法,只接受.txt文件")
            return

        # 根据文件路径 获取文件名
        inp_file_name = get_filename(inp_file, suffix=True)

        self.__logger.info(f"正在读取,需要去重的文件总行数...{inp_file_name}")
        line_count, duration_time = get_file_line_cnt(inp_file, duration=True)
        self.__logger.info(f"读取完成,总行数为:{line_count};耗时:{duration_time}秒...{inp_file_name}")

        # 创建输出目录
        order_id = get_orderId_random()
        out_dir = os.path.join(out_dir, order_id)
        num_shards = line_count // 10000
        num_shards = 1 if num_shards <= 0 else num_shards
        num_shards = 3000 if num_shards >= 3000 else num_shards
        self.__logger.info(f"临时文件切片为:{num_shards}")

        shard_path_list = []  # 存储切片路径
        shard_file_obj_list = []  # 存储切片文件对象
        for _ in range(num_shards):
            shard_path = f"{os.path.join(out_dir, f'{_}.tmp')}"
            mkdirs_dir(shard_path)
            shard_path_list.append(shard_path)
            shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))

        # 写入数据
        with open(inp_file, "r", encoding="utf-8") as f_r:
            tqdm_f = tqdm(f_r, total=line_count, desc="正在去重-写入切片-步骤(1/2)",
                          bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
            for line_i in tqdm_f:
                line = line_i.strip().encode()
                line_hash = hashlib.md5(line).hexdigest()
                shard_id = int(line_hash, 16) % num_shards
                shard_file_obj_list[shard_id].write(line_i)

        # 关闭切片文件对象
        for shard_file_obj in shard_file_obj_list:
            shard_file_obj.close()

        # 根据每个切片去去重
        out_file_name = inp_file_name if not out_file_name else out_file_name
        result_w_path = os.path.join(out_dir, out_file_name)
        tqdm_f = tqdm(shard_path_list, total=len(shard_path_list), desc="正在去重-去重切片-步骤(2/2)",
                      bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
        with open(result_w_path, "w", encoding="utf-8") as f_w:
            for shard_path in tqdm_f:
                with open(shard_path, "r", encoding="utf-8") as f_r:
                    seen_list = []
                    for line_i in f_r.readlines():
                        line = line_i.strip()
                        seen_list.append(line)
                    seen_list = list(set(seen_list))
                    if seen_list:
                        w_txt = "\n".join(seen_list)
                        f_w.write(w_txt + "\n")
                os.remove(shard_path)  # 删除临时文件

        # 读取结果文件 获取去重后的数据
        with open(result_w_path, "r", encoding="utf-8") as fp_r:
            line_count = sum(1 for _ in fp_r)
        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")


if __name__ == '__main__':
    FileDeduper().dedup_file(inp_file=r"D:\000\xtnkk-tools\demos\113.txt",
                             out_dir=r"D:\000\xtnkk-tools\demos\data")
