#!/usr/bin/env python
# -*- coding: utf-8 -*-

# 说明：
#    装对Phone文件做去重
# History:
# Date          Author    Version       Modification
# --------------------------------------------------------------------------------------------------
# 2026/2/2    xiatn     V00.01.000    新建
# --------------------------------------------------------------------------------------------------
import os
import fnmatch
import hashlib
from tqdm import tqdm
from xtn_tools_pro.utils.log import Log
from xtn_tools_pro.utils.helpers import get_orderId_random
from xtn_tools_pro.utils.file_utils import get_file_line_cnt, get_filename, mkdirs_dir, get_file_extension, is_dir, \
    get_listdir


class FileDeduper:
    def __init__(self):
        self.__logger = Log('set_data', './xxx.log', log_level='DEBUG', is_write_to_console=True,
                            is_write_to_file=False,
                            color=True, mode='a', save_time_log_path='./logs')

    def dedup_file(self, inp_file, out_dir, out_file_name=None):
        """
            对单文件去重
        :param inp_file: 需要去重的文件
        :param out_dir:去重后文件保存在哪个目录下,如果目录不存在则会自动创建
        :param out_file_name:去重后保存的文件名称,如果不传 默认使用的是 <inp_file> 的文件名
        :return:
        """
        if get_file_extension(inp_file) != ".txt":
            self.__logger.critical("文件不合法,只接受.txt文件")
            return

        # 根据文件路径 获取文件名
        inp_file_name = get_filename(inp_file, suffix=True)

        self.__logger.info(f"正在读取,需要去重的文件总行数...{inp_file_name}")
        line_count, duration_time = get_file_line_cnt(inp_file, duration=True)
        self.__logger.info(f"读取完成,总行数为:{line_count};耗时:{duration_time}秒...{inp_file_name}")

        # 创建输出目录
        order_id = get_orderId_random()
        out_dir = os.path.join(out_dir, order_id)
        num_shards = line_count // 10000
        num_shards = 1 if num_shards <= 0 else num_shards
        num_shards = 3000 if num_shards >= 3000 else num_shards
        self.__logger.info(f"临时文件切片为:{num_shards}")

        shard_path_list = []  # 存储切片路径
        shard_file_obj_list = []  # 存储切片文件对象
        for _ in range(num_shards):
            shard_path = f"{os.path.join(out_dir, f'{_}.tmp')}"
            mkdirs_dir(shard_path)
            shard_path_list.append(shard_path)
            shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))

        # 写入数据
        with open(inp_file, "r", encoding="utf-8") as f_r:
            tqdm_f = tqdm(f_r, total=line_count, desc="正在去重-写入切片-步骤(1/2)",
                          bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
            for line_i in tqdm_f:
                line = line_i.strip().encode()
                line_hash = hashlib.md5(line).hexdigest()
                shard_id = int(line_hash, 16) % num_shards
                shard_file_obj_list[shard_id].write(line_i)

        # 关闭切片文件对象
        for shard_file_obj in shard_file_obj_list:
            shard_file_obj.close()

        # 根据每个切片去去重
        out_file_name = inp_file_name if not out_file_name else out_file_name
        result_w_path = os.path.join(out_dir, out_file_name)
        tqdm_f = tqdm(shard_path_list, total=len(shard_path_list), desc="正在去重-处理切片-步骤(2/2)",
                      bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
        with open(result_w_path, "w", encoding="utf-8") as f_w:
            for shard_path in tqdm_f:
                with open(shard_path, "r", encoding="utf-8") as f_r:
                    seen_list = []
                    for line_i in f_r.readlines():
                        line = line_i.strip()
                        seen_list.append(line)
                    seen_list = list(set(seen_list))
                    if seen_list:
                        w_txt = "\n".join(seen_list)
                        f_w.write(w_txt + "\n")
                os.remove(shard_path)  # 删除临时文件

        # 读取结果文件 获取去重后的数据
        line_count, duration_time = get_file_line_cnt(result_w_path, duration=True)
        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")

    def dedup_dir(self, inp_dir, out_dir, out_file_name=None):
        """
            对文件夹下的所有txt文件一起去重,最终保存为一个文件
        :param inp_dir: 需要去重的文件夹
        :param out_dir:去重后文件保存在哪个目录下,如果目录不存在则会自动创建
        :param out_file_name:去重后保存的文件名称,如果不传 默认使用的是 <000_对文件夹去重_结果> 的文件名
        :return:
        """
        if not is_dir(inp_dir):
            self.__logger.critical("文件夹不存在或不合法")
            return

        self.__logger.info("正在统计可去重文件数量...")
        inp_file_path_list = []
        for set_file_name in get_listdir(inp_dir):
            if fnmatch.fnmatch(set_file_name, '*.txt'):
                inp_file_path_list.append(os.path.join(inp_dir, set_file_name))
        self.__logger.info(f"当前文件夹下可去重文件数量为:{len(inp_file_path_list)}")

        if len(inp_file_path_list) <= 0:
            return

        order_id = get_orderId_random()
        out_dir = os.path.join(out_dir, order_id)
        num_shards = 3000
        self.__logger.info(f"临时文件切片为:{num_shards}")

        shard_path_list = []  # 存储切片路径
        shard_file_obj_list = []  # 存储切片文件对象
        for _ in range(num_shards):
            shard_path = f"{os.path.join(out_dir, f'{_}.tmp')}"
            mkdirs_dir(shard_path)
            shard_path_list.append(shard_path)
            shard_file_obj_list.append(open(shard_path, "w", encoding="utf-8"))

        # 写入数据
        for _ in range(len(inp_file_path_list)):
            inp_file = inp_file_path_list[_]
            # 根据文件路径 获取文件名
            inp_file_name = get_filename(inp_file, suffix=True)
            # self.__logger.info(f"正在读取,需要去重的文件总行数...{inp_file_name}")
            line_count, duration_time = get_file_line_cnt(inp_file, duration=True)
            # self.__logger.info(f"读取完成,总行数为:{line_count};耗时:{duration_time}秒...{inp_file_name}")

            with open(inp_file, "r", encoding="utf-8") as f_r:
                tqdm_f = tqdm(f_r, total=line_count, desc=f"正在去重-写入切片-步骤({_ + 1}/{len(inp_file_path_list) + 1})",
                              bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
                for line_i in tqdm_f:
                    line = line_i.strip().encode()
                    line_hash = hashlib.md5(line).hexdigest()
                    shard_id = int(line_hash, 16) % num_shards
                    shard_file_obj_list[shard_id].write(line_i)

        for shard_file_obj in shard_file_obj_list:
            shard_file_obj.close()

        # 根据每个切片去去重
        out_file_name = "000_对文件夹去重_结果.txt" if not out_file_name else out_file_name
        result_w_path = os.path.join(out_dir, out_file_name)
        tqdm_f = tqdm(shard_path_list, total=len(shard_path_list),
                      desc=f"正在去重({len(inp_file_path_list) + 1}/{len(inp_file_path_list) + 1})",
                      bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")

        with open(result_w_path, "w", encoding="utf-8") as f_w:
            for shard_path in tqdm_f:
                with open(shard_path, "r", encoding="utf-8") as f_r:
                    seen_list = []
                    for line_i in f_r.readlines():
                        line = line_i.strip()
                        seen_list.append(line)
                    seen_list = list(set(seen_list))
                    if seen_list:
                        w_txt = "\n".join(seen_list)
                        f_w.write(w_txt + "\n")
                os.remove(shard_path)  # 删除临时文件

        # 读取结果文件 获取去重后的数据
        line_count, duration_time = get_file_line_cnt(result_w_path, duration=True)
        self.__logger.info(f"文件处理完毕,去重后总行数为:{line_count},结果路径:{result_w_path}")

    def merging_dir(self, inp_dir, out_dir, out_file_name=None):
        """
            指定文件夹合并该文件夹下的所有 .txt 文件
            仅合并,不去重,不打乱!
            仅合并,不去重,不打乱!
            仅合并,不去重,不打乱!
        :param inp_dir: 需要合并的文件夹
        :param out_dir:合并后文件保存在哪个目录下,如果目录不存在则会自动创建
        :param out_file_name:合并后保存的文件名称,如果不传 默认使用的是 <000_对文件夹合并_结果> 的文件名
        :return:
        """
        if not is_dir(inp_dir):
            self.__logger.critical("文件夹不存在或不合法")
            return

        self.__logger.info("正在统计可合并文件数量...")
        inp_file_path_list = []
        for set_file_name in get_listdir(inp_dir):
            if fnmatch.fnmatch(set_file_name, '*.txt'):
                inp_file_path_list.append(os.path.join(inp_dir, set_file_name))
        self.__logger.info(f"当前文件夹下可合并文件数量为:{len(inp_file_path_list)}")

        if len(inp_file_path_list) <= 0:
            return

        order_id = get_orderId_random()
        out_dir = os.path.join(out_dir, order_id)
        out_file_name = "000_对文件夹合并_结果.txt" if not out_file_name else out_file_name
        result_w_path = os.path.join(out_dir, out_file_name)
        mkdirs_dir(result_w_path)
        with open(result_w_path, "w", encoding="utf-8") as f_w:
            for _ in range(len(inp_file_path_list)):
                inp_file = inp_file_path_list[_]
                inp_file_name = get_filename(inp_file, suffix=True)
                # self.__logger.info(f"正在读取,需要去重的文件总行数...{inp_file_name}")
                line_count, duration_time = get_file_line_cnt(inp_file, duration=True)
                # self.__logger.info(f"读取完成,总行数为:{line_count};耗时:{duration_time}秒...{inp_file_name}")

                with open(inp_file, "r", encoding="utf-8") as f_r:
                    tqdm_f = tqdm(f_r, total=line_count,
                                  desc=f"正在合并文件({_ + 1}/{len(inp_file_path_list)})",
                                  bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
                    for line_i in tqdm_f:
                        line = line_i.strip()
                        f_w.write(line + "\n")

        # 读取结果文件 获取合并后的数据
        line_count, duration_time = get_file_line_cnt(result_w_path, duration=True)
        self.__logger.info(f"文件处理完毕,合并后总行数为:{line_count},结果路径:{result_w_path}")

    def split_file(self, inp_file, out_dir, out_file_name, start_index=1, max_line=1000000):
        """
            分割文件
        :param inp_file: 需要分割的文件
        :param out_dir: 分割后文件保存在哪个目录下,如果目录不存在则会自动创建
        :param out_file_name: 分割后保存的文件名称,如果不传 默认使用的是 <inp_file> 的文件名
        :param start_index: 分割文件开始序号
        :param max_line: 每个文件多少行
        :return:
        """
        if get_file_extension(inp_file) != ".txt":
            self.__logger.critical("文件不合法,只接受.txt文件")
            return

        # 根据文件路径 获取文件名
        inp_file_name = get_filename(inp_file, suffix=True)
        out_file_name = inp_file_name if not out_file_name else out_file_name

        order_id = get_orderId_random()
        out_dir = os.path.join(out_dir, order_id)

        self.__logger.info(f"正在读取,需要分割的文件总行数...{inp_file_name}")
        line_count, duration_time = get_file_line_cnt(inp_file, duration=True)
        self.__logger.info(f"读取完成,总行数为:{line_count};耗时:{duration_time}秒...{inp_file_name}")

        with open(inp_file, "r", encoding="utf-8") as f_r:
            tqdm_f = tqdm(f_r, total=line_count, desc="正在分割-步骤(1/1)",
                          bar_format="{l_bar}{bar}|{n}/{total} [预计完成时间:{remaining}]")
            temp_line_list = []
            for line_i in tqdm_f:
                line = line_i.strip()
                temp_line_list.append(line)
                if len(temp_line_list) == max_line:
                    result_w_path = os.path.join(out_dir, f"{start_index}_{out_file_name}")
                    self.__list_to_write_file(result_w_path, temp_line_list)
                    temp_line_list = []
                    start_index += 1
            if temp_line_list:
                result_w_path = os.path.join(out_dir, f"{start_index}_{out_file_name}")
                self.__list_to_write_file(result_w_path, temp_line_list)

        split_file_cnt = len(get_listdir(out_dir))
        self.__logger.info(f"文件处理完毕,分割后总文件数为:{split_file_cnt},结果路径:{out_dir}")

    def __list_to_write_file(self, write_path, data_list):
        """
            根据列表数据 写入 指定文件中
        :param write_path: 文件路径
        :param data_list: 数据列表
        :return:
        """
        if not data_list: return
        mkdirs_dir(write_path)
        with open(write_path, "w", encoding="utf-8") as result_w_f:
            result_w_f.write("\n".join(data_list))


if __name__ == '__main__':
    # 对单文件去重
    # FileDeduper().dedup_file(inp_file=r"D:\000\xtnkk-tools\demos\113.txt",
    #                          out_dir=r"D:\000\xtnkk-tools\demos\data")

    # 对文件夹去重
    # FileDeduper().dedup_dir(inp_dir=r"D:\000\xtnkk-tools\demos",
    #                         out_dir=r"D:\000\xtnkk-tools\demos\data",
    #                         out_file_name="demos.txt")

    # 对文件夹合并
    # FileDeduper().merging_dir(inp_dir=r"D:\000\xtnkk-tools\demos",
    #                           out_dir=r"D:\000\xtnkk-tools\demos\data",
    #                           out_file_name="合并数据.txt")

    # 分割文件
    FileDeduper().split_file(inp_file=r"D:\000\xtnkk-tools\demos\data\000_对文件夹合并_结果.txt",
                             out_dir=r"D:\000\xtnkk-tools\demos\data",
                             out_file_name="切割数据.txt")
