"""Filter and deduplicate reconstructed article JSONL files.

Reads JSONL files produced by the reconstruction step, filters articles
with a Boolean query, and writes a single deduplicated JSONL output.

Query syntax:
    Operators: AND, OR, NOT (case-insensitive)
    Parentheses: (...) to group
    Terms: single words or quoted phrases ("giorgia meloni")
"""

from __future__ import annotations

import json
import os
import re
from typing import Any, Dict, List, Optional, Tuple

try:
    import boolean  # type: ignore
except ImportError:
    boolean = None  # type: ignore[assignment]


# --- Boolean query parsing ---

_algebra = None


def _get_algebra():
    global _algebra
    if _algebra is None:
        if boolean is None:
            raise ImportError("Install boolean.py: pip install boolean.py")
        _algebra = boolean.BooleanAlgebra()
    return _algebra


def _normalize_ops(query: str) -> str:
    q = re.sub(r"\bAND\b", " and ", query, flags=re.IGNORECASE)
    q = re.sub(r"\bOR\b", " or ", q, flags=re.IGNORECASE)
    q = re.sub(r"\bNOT\b", " not ", q, flags=re.IGNORECASE)
    return re.sub(r"\s+", " ", q).strip()


def _extract_phrases(query: str) -> Tuple[str, Dict[str, str]]:
    phrases: Dict[str, str] = {}

    def repl(match: re.Match) -> str:
        key = f"PHRASE_{len(phrases)}"
        phrases[key] = match.group(1)
        return key

    return re.sub(r'"([^"]+)"', repl, query), phrases


def build_query_expr(query: Optional[str]):
    if query is None or not str(query).strip():
        return None, {}
    q, phrases = _extract_phrases(query)
    q = _normalize_ops(q)
    algebra = _get_algebra()
    try:
        expr = algebra.parse(q)
    except Exception as exc:
        raise ValueError(f"Invalid Boolean query: {exc}") from exc
    return expr, phrases


def text_matches_query(text: str, expr: Any, phrases: Dict[str, str]) -> bool:
    if expr is None:
        return True
    text_cf = text.casefold()
    algebra = _get_algebra()
    subs: Dict[Any, Any] = {}
    for sym in expr.get_symbols():
        name = sym.obj
        term = phrases[name].casefold() if name in phrases else str(name).casefold()
        subs[sym] = algebra.TRUE if term in text_cf else algebra.FALSE
    value = expr.subs(subs).simplify()
    return value == algebra.TRUE


# --- JSONL filtering and deduplication ---

def _iter_jsonl_files(input_dir: str) -> List[str]:
    files = []
    for name in os.listdir(input_dir):
        if name.lower().endswith(".jsonl"):
            files.append(os.path.join(input_dir, name))
    files.sort()
    return files


def filtermerge(
    input_dir: str,
    output_file: str,
    *,
    query: Optional[str] = None,
    verbose: bool = True,
) -> int:
    """Filter JSONL files and write one deduplicated JSONL output.

    Args:
        input_dir: Directory containing per-file JSONL files.
        output_file: Final output JSONL path.
        query: Boolean query string, or None for no filtering.
        verbose: Print progress messages.

    Returns:
        Number of unique articles in the output.
    """
    expr, phrases = build_query_expr(query)
    jsonl_files = _iter_jsonl_files(input_dir)

    if not jsonl_files:
        raise ValueError(f"No JSONL files found in: {input_dir}")

    if verbose:
        print(f"Processing {len(jsonl_files)} JSONL files...")

    # Deduplicate by URL, keeping the longest text
    best: Dict[str, Dict] = {}

    for path in jsonl_files:
        with open(path, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                try:
                    article = json.loads(line)
                except json.JSONDecodeError:
                    continue

                text = article.get("text", "")
                url = article.get("url", "")
                if not url:
                    continue

                if not text_matches_query(text, expr, phrases):
                    continue

                prev = best.get(url)
                if prev is None or len(text) > len(prev.get("text", "")):
                    best[url] = article

    # Write output
    with open(output_file, "w", encoding="utf-8") as f:
        for article in best.values():
            f.write(json.dumps(article, ensure_ascii=False) + "\n")

    count = len(best)
    if verbose:
        print(f"Wrote {count} unique articles to {output_file}")
    return count


__all__ = ["filtermerge", "build_query_expr"]
