import itertools
import json
import os
import random
import re
from typing import Any
from json_dataset_builder import extract_sophora_text, clean_html

"""
Description:
    This script merges the contents of JSON files from a specified directory into new JSON files.
    Unlike sequential merging, it uses a combinatorial approach:
    it generates all possible combinations of the source files based on a given group size (step).
    Each combination is saved as a separate JSON file.

Core Idea:
    - Let n = total number of source JSON files.
    - Let k = group size (step).
    - The number of possible combinations is calculated using the binomial coefficient formula:
        C(n, k) = n! / (k! * (n - k)!)
    - For example, if n = 75 and k = 10, the number of combinations will be extremely large,
      so resource limitations should be considered.

Warning:
    - The number of combinations grows exponentially with step.
    - For large n and k, consider limiting the number of combinations or using random sampling.
"""


import itertools
import random

def merge2(step: int = 2, limit: int = 200):
    json_paths = load_json_files()
    combos = itertools.combinations(json_paths, step)

    # Reservoir sampling
    reservoir = []
    for i, combo in enumerate(combos, start=1):
        if len(reservoir) < limit:
            reservoir.append(combo)
        else:
            j = random.randint(1, i)
            if j <= limit:
                reservoir[j - 1] = combo

    # Теперь reservoir содержит случайные 200 комбинаций
    for idx, combo in enumerate(reservoir):
        merged = {"source_files": [], "file_contents": []}
        for path in combo:
            json_i = load_json_file(path)
            if has_defect(json_i):
                continue
            merged["source_files"].append(get_filename(path))
            merged["file_contents"].append(parse_json(json_i))

        write_to_file(idx, merged)

def get_filename(path: str) -> str:
    return re.search(r'[^/]+$', path).group(0)

def load_json_files():
    input_folder = "/home/nurseiit/PycharmProjects/wdr-article-semantic-chunking2/Artikel_WDR_NRW"

    files = sorted([f for f in os.listdir(input_folder) if f.endswith('.json')])
    res_files = [os.path.join(input_folder, f) for f in files]
    if len(res_files) < 2:
        raise Exception("Not enough JSON files to process.")
    return res_files



def load_json_file(path):
    with open(path, 'r', encoding='utf-8') as f:
        return json.load(f)



def create_file_name(json_paths: list[str], t: int) -> str:
    return re.search(r'[^/]+$', json_paths[t]).group(0)


def write_to_file(i: int, merged: dict[str, list[Any]]):
    output_filename = f'merged_filtered_{i}.json'
    from pathlib import Path
    folder_name = "data"
    file_name = f"{output_filename}"
    path = Path.cwd() / folder_name / file_name
    path.parent.mkdir(parents=True, exist_ok=True)

    with open(path, 'w', encoding='utf-8') as outfile:
        json.dump(merged, outfile, ensure_ascii=False, indent=4)


def parse_json(json):
    res_json = {}
    res_json["sophora:tags"] = clean_html(json.get("sophora:tags"))
    res_json["wdr:leadin"] = clean_html(json.get("wdr:leadin"))
    res_json["wdr:schlagzeile"] = clean_html(json.get("wdr:schlagzeile"))
    res_json["wdr:teasertext"] = clean_html(json.get("wdr:teasertext"))
    res_json["sophora-extension:text"] = clean_html(extract_sophora_text(json.get("wdr:text", [])))
    return res_json


def has_defect(json_file) -> bool:
    sophora_text = extract_sophora_text(json_file.get("wdr:text", []))
    if len(sophora_text) < 4:
        return True
    else:
        return False


if __name__ == "__main__":
    merge2(4,200)
