"""Heuristic filters for trimming HTML/Markdown before LLM cleanup.

These helpers target the recurring boilerplate blocks that dominate the
estate-planning KB crawl (cookie banners, CTAs, recommendations, etc.).
The goal is to shrink documents before they reach `remove_boilerplate_batch`.
"""

from __future__ import annotations

import re
from dataclasses import dataclass
from typing import List

from bs4 import BeautifulSoup, Tag

from .ingest_markdown import link_percentage


HTML_DROP_TAGS = (
    "script",
    "style",
    "noscript",
    "svg",
    "iframe",
    "footer",
    "header",
    "nav",
    "aside",
    "form",
)

HTML_ATTR_KEYWORDS = (
    "cookie",
    "consent",
    "privacy",
    "gdpr",
    "banner",
    "promo",
    "subscribe",
    "newsletter",
    "sign-up",
    "signup",
    "share",
    "social",
    "related",
    "recommend",
    "nav",
    "breadcrumb",
    "toc",
    "legal dictionary",
)


def clean_ep_html(html: str) -> str:
    """Strip obvious boilerplate nodes before markdown conversion."""

    soup = BeautifulSoup(html, "html.parser")

    for tag in HTML_DROP_TAGS:
        for node in soup.find_all(tag):
            node.decompose()

    # Remove elements whose attributes mention common boilerplate keywords.
    for node in list(soup.find_all(True)):
        if not isinstance(node, Tag):
            continue
        attributes: list[str] = []
        node_attrs = node.attrs or {}
        for attr in ("class", "id", "aria-label", "role"):
            value = node_attrs.get(attr)
            if value is None:
                continue
            if isinstance(value, str):
                attributes.append(value)
            else:
                attributes.extend(value)

        attr_blob = " ".join(attributes).lower()
        if attr_blob and any(keyword in attr_blob for keyword in HTML_ATTR_KEYWORDS):
            node.decompose()

    return str(soup)


SECTION_HEADING_PATTERNS = [
    r"privacy",
    r"cookie",
    r"consent",
    r"opt[- ]out",
    r"advertising",
    r"social media",
    r"recommended",
    r"related",
    r"similar",
    r"see also",
    r"in this article",
    r"learn more",
    r"want more legal info",
    r"make (?:your|a) .*free",
    r"donation",
    r"references",
    r"notes",
    r"the bottom line",
    r"next steps",
    r"external links",
    r"book",
    r"ebook",
    r"get started",
]

SECTION_HEADING_REGEXES = [
    re.compile(pat, re.IGNORECASE) for pat in SECTION_HEADING_PATTERNS
]

SECTION_BODY_PATTERNS = [
    r"cookie list",
    r"manage consent",
    r"###?\s*recommended",
    r"share this article",
    r"start typing, hit enter",
    r"\benz\b cookies",
]

SECTION_BODY_REGEXES = [re.compile(pat, re.IGNORECASE) for pat in SECTION_BODY_PATTERNS]

LINE_EXACT_MATCHES = {
    "label",
    "<img>",
    "<iframe></iframe>",
    "recommended for you",
    "get started",
    "create your estate plan today.",
    "estate planning \u20144 min read",
    "estate planning \u201410 min read",
    "online now",
    "book & ebook",
    "view all articles",
}

LINE_REGEXES = [
    re.compile(r"^[-]{1,4}$"),
    re.compile(r"^\s*[-+*]\s*share this article", re.IGNORECASE),
    re.compile(r"^\s*recommended for you", re.IGNORECASE),
    re.compile(r"^\s*create your .*estate plan", re.IGNORECASE),
    re.compile(r"estate planning \u2014\d+ min read", re.IGNORECASE),
    re.compile(r"^\s*get started$", re.IGNORECASE),
]


HEADING_RE = re.compile(r"^\s{0,3}(#{1,6})\s+(.*)$")


@dataclass
class Section:
    heading: str | None
    level: int
    lines: List[str]

    @property
    def text(self) -> str:
        return "\n".join(self.lines).strip("\n")


def _split_into_sections(markdown: str) -> List[Section]:
    sections: List[Section] = []
    current = Section(heading=None, level=0, lines=[])

    for line in markdown.splitlines():
        match = HEADING_RE.match(line)
        if match:
            sections.append(current)
            level = len(match.group(1))
            heading = match.group(2).strip()
            current = Section(heading=heading, level=level, lines=[line])
        else:
            current.lines.append(line)

    sections.append(current)
    return sections


def _should_drop_section(section: Section) -> bool:
    if not section.lines:
        return True

    raw_text = section.text
    if not raw_text.strip():
        return True

    heading = section.heading or ""

    for regex in SECTION_HEADING_REGEXES:
        if heading and regex.search(heading):
            return True

    for regex in SECTION_BODY_REGEXES:
        if regex.search(raw_text):
            return True

    ratio = link_percentage(raw_text)
    if heading and ratio > 0.45 and len(raw_text) < 2000:
        return True

    return False


def _remove_noise_lines(text: str) -> str:
    lines = text.splitlines()
    kept: List[str] = []

    for line in lines:
        stripped = line.strip()
        if not stripped:
            kept.append("")
            continue

        if stripped.lower() in LINE_EXACT_MATCHES:
            continue

        if any(regex.search(stripped) for regex in LINE_REGEXES):
            continue

        kept.append(line)

    cleaned = "\n".join(kept)
    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
    return cleaned.strip()


def filter_ep_markdown(markdown: str) -> str:
    """Apply heuristic filters to markdown derived from estate-plan pages."""

    sections = _split_into_sections(markdown)
    kept_sections: List[str] = []

    for section in sections:
        if _should_drop_section(section):
            continue
        kept_sections.append(section.text)

    filtered = "\n\n".join(chunk for chunk in kept_sections if chunk.strip())
    return _remove_noise_lines(filtered)


__all__ = ["clean_ep_html", "filter_ep_markdown"]
