"""HTML-based Yahoo Finance news scraping with structured-data fallbacks."""

from __future__ import annotations

import json
import logging
import re
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from typing import Any
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup, Tag

LOGGER = logging.getLogger(__name__)

DEFAULT_TIMEOUT = 10
DEFAULT_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/131.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9,de;q=0.8",
}
DEFAULT_BASE_URL = "https://finance.yahoo.com"
GERMAN_BASE_URL = "https://de.finance.yahoo.com"

ROOT_APP_MAIN_MARKER = "root.App.main"
UUID_PATTERN = re.compile(r"g:([a-f0-9-]{36})", re.IGNORECASE)
RELATIVE_TIME_PATTERN = re.compile(
    r"""
    ^\s*
    (?:
        (?P<english_value>\d+)\s*
        (?P<english_unit>d(?:ays?)?|h(?:ours?)?|hrs?|m(?:in(?:ute)?s?)?)\s*ago
        |
        vor\s+(?P<german_value>\d+)\s*
        (?P<german_unit>tag(?:en)?|tage|stunde(?:n)?|std\.?|minute(?:n)?|min\.?)
        |
        (?P<compact_value>\d+)\s*(?P<compact_unit>d|h|m)\s*ago
    )
    \s*$
    """,
    re.IGNORECASE | re.VERBOSE,
)


def _utc_now() -> datetime:
    return datetime.now(timezone.utc)


def _as_mapping(value: Any) -> Mapping[str, Any] | None:
    return value if isinstance(value, Mapping) else None


def _as_sequence(value: Any) -> Sequence[Any] | None:
    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
        return value
    return None


def _coalesce(*values: Any) -> Any:
    for value in values:
        if value not in (None, "", [], {}):
            return value
    return None


def _clean_text(value: Any) -> str:
    if value is None:
        return ""
    return str(value).strip()


def _normalize_url(url: Any, base_url: str) -> str:
    if not isinstance(url, str):
        return ""
    cleaned = url.strip()
    if not cleaned:
        return ""
    return urljoin(base_url, cleaned)


def _extract_json_object(text: str, marker: str) -> str | None:
    marker_index = text.find(marker)
    if marker_index == -1:
        return None

    start_index = text.find("{", marker_index)
    if start_index == -1:
        return None

    depth = 0
    in_string = False
    quote_char = ""
    escaped = False

    for index in range(start_index, len(text)):
        char = text[index]
        if in_string:
            if escaped:
                escaped = False
            elif char == "\\":
                escaped = True
            elif char == quote_char:
                in_string = False
            continue

        if char in {"'", '"'}:
            in_string = True
            quote_char = char
            continue

        if char == "{":
            depth += 1
            continue

        if char == "}":
            depth -= 1
            if depth == 0:
                return text[start_index : index + 1]

    return None


def _iter_mappings(value: Any, depth: int = 0, max_depth: int = 12):
    if depth > max_depth:
        return

    mapping = _as_mapping(value)
    if mapping is not None:
        yield mapping
        for child in mapping.values():
            yield from _iter_mappings(child, depth=depth + 1, max_depth=max_depth)
        return

    sequence = _as_sequence(value)
    if sequence is not None:
        for child in sequence:
            yield from _iter_mappings(child, depth=depth + 1, max_depth=max_depth)


def _looks_like_article(candidate: Mapping[str, Any]) -> bool:
    for payload in (candidate, _as_mapping(candidate.get("content"))):
        if payload is None:
            continue

        title = _coalesce(payload.get("title"), payload.get("headline"), payload.get("name"))
        url = _extract_candidate_url(payload, DEFAULT_BASE_URL)
        if title and url:
            return True

    return False


def _extract_candidate_url(candidate: Mapping[str, Any], base_url: str) -> str:
    click_through = _as_mapping(candidate.get("clickThroughUrl"))
    canonical = _as_mapping(candidate.get("canonicalUrl"))
    preview = _as_mapping(candidate.get("previewUrl"))

    raw_url = _coalesce(
        candidate.get("link"),
        candidate.get("url"),
        click_through.get("url") if click_through else None,
        canonical.get("url") if canonical else None,
        preview.get("url") if preview else None,
    )
    return _normalize_url(raw_url, base_url)


def _to_relative_delta(unit: str, value: int) -> timedelta | None:
    if unit.startswith(("d", "tag")):
        return timedelta(days=value)
    if unit.startswith(("h", "st", "std")):
        return timedelta(hours=value)
    if unit.startswith(("m", "min")):
        return timedelta(minutes=value)
    return None


def _parse_relative_time(time_text: str) -> datetime | None:
    normalized = _clean_text(time_text).lower().rstrip(".")
    if not normalized:
        return None

    if normalized in {"yesterday", "gestern"}:
        return _utc_now() - timedelta(days=1)

    match = RELATIVE_TIME_PATTERN.match(normalized)
    if not match:
        return None

    value_text = _coalesce(
        match.group("english_value"),
        match.group("german_value"),
        match.group("compact_value"),
    )
    unit_text = _coalesce(
        match.group("english_unit"),
        match.group("german_unit"),
        match.group("compact_unit"),
    )

    if value_text is None or unit_text is None:
        return None

    value = int(value_text)
    delta = _to_relative_delta(unit_text.lower(), value)
    return _utc_now() - delta if delta is not None else None


def _parse_publish_time(value: Any) -> float:
    if isinstance(value, (int, float)):
        timestamp = float(value)
        if timestamp > 10_000_000_000:
            timestamp /= 1000
        return timestamp

    text = _clean_text(value)
    if not text:
        return _utc_now().timestamp()

    if text.isdigit():
        return _parse_publish_time(int(text))

    relative = _parse_relative_time(text)
    if relative is not None:
        return relative.timestamp()

    try:
        parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
    except ValueError:
        try:
            parsed = parsedate_to_datetime(text)
        except (TypeError, ValueError):
            return _utc_now().timestamp()

    if parsed.tzinfo is None:
        parsed = parsed.replace(tzinfo=timezone.utc)
    return parsed.astimezone(timezone.utc).timestamp()


def _extract_related_tickers(*payloads: Mapping[str, Any] | None, default_symbol: str) -> list[str]:
    related: list[str] = []

    def add_symbol(raw_symbol: Any) -> None:
        symbol = ""
        if isinstance(raw_symbol, str):
            symbol = raw_symbol
        elif isinstance(raw_symbol, Mapping):
            symbol = _clean_text(
                _coalesce(
                    raw_symbol.get("symbol"),
                    raw_symbol.get("tickerSymbol"),
                    raw_symbol.get("id"),
                    raw_symbol.get("displayName"),
                )
            )
        symbol = symbol.strip().upper()
        if symbol and symbol not in related:
            related.append(symbol)

    for payload in payloads:
        if payload is None:
            continue

        for key in ("relatedTickers", "symbols", "tickers"):
            sequence = _as_sequence(payload.get(key))
            if sequence is not None:
                for item in sequence:
                    add_symbol(item)

        finance = _as_mapping(payload.get("finance"))
        if finance is not None:
            for key in ("stockTickers", "symbols"):
                sequence = _as_sequence(finance.get(key))
                if sequence is not None:
                    for item in sequence:
                        add_symbol(item)

    return related or [default_symbol.upper()]


def _extract_thumbnail(*payloads: Mapping[str, Any] | None) -> dict[str, Any]:
    resolutions: list[dict[str, str]] = []

    def add_url(raw_url: Any) -> None:
        url = _clean_text(raw_url)
        if not url:
            return
        entry = {"url": url}
        if entry not in resolutions:
            resolutions.append(entry)

    for payload in payloads:
        if payload is None:
            continue

        thumbnail = payload.get("thumbnail")
        if isinstance(thumbnail, str):
            add_url(thumbnail)
            continue

        thumbnail_mapping = _as_mapping(thumbnail)
        if thumbnail_mapping is None:
            continue

        add_url(_coalesce(thumbnail_mapping.get("originalUrl"), thumbnail_mapping.get("url")))
        for resolution in _as_sequence(thumbnail_mapping.get("resolutions")) or []:
            resolution_mapping = _as_mapping(resolution)
            if resolution_mapping is not None:
                add_url(resolution_mapping.get("url"))

    return {"resolutions": resolutions} if resolutions else {}


def _extract_uuid(
    *payloads: Mapping[str, Any] | None,
    fallback_link: str,
    fallback_title: str,
    symbol: str,
) -> str:
    for payload in payloads:
        if payload is None:
            continue

        for key in ("uuid", "id", "articleId"):
            candidate = _clean_text(payload.get(key))
            if candidate:
                return candidate

        data_ylk = _clean_text(payload.get("data-ylk"))
        match = UUID_PATTERN.search(data_ylk)
        if match:
            return match.group(1)

    stable_value = fallback_link or f"{symbol}:{fallback_title}"
    return str(uuid.uuid5(uuid.NAMESPACE_URL, stable_value))


@dataclass(frozen=True)
# pylint: disable=too-many-instance-attributes
class NewsItem:
    uuid: str
    title: str
    link: str
    publisher: str
    provider_publish_time: float
    article_type: str
    related_tickers: list[str]
    thumbnail: dict[str, Any]
    summary: str

    def to_dict(self) -> dict[str, Any]:
        return {
            "uuid": self.uuid,
            "title": self.title,
            "link": self.link,
            "publisher": self.publisher,
            "providerPublishTime": self.provider_publish_time,
            "type": self.article_type,
            "relatedTickers": self.related_tickers,
            "thumbnail": self.thumbnail,
            "summary": self.summary,
        }


def _normalize_article(candidate: Mapping[str, Any], symbol: str, base_url: str) -> NewsItem | None:
    content = _as_mapping(candidate.get("content"))
    article = content or candidate

    title = _clean_text(
        _coalesce(article.get("title"), article.get("headline"), article.get("name"))
    )
    link = _extract_candidate_url(article, base_url)
    if not title or not link:
        return None

    provider = _clean_text(
        _coalesce(
            article.get("publisher"),
            article.get("providerName"),
            (_as_mapping(article.get("provider")) or {}).get("displayName"),
            (_as_mapping(article.get("publisher")) or {}).get("name"),
        )
    ) or "Unknown"

    summary = _clean_text(_coalesce(article.get("summary"), article.get("description")))
    publish_time = _parse_publish_time(
        _coalesce(
            article.get("providerPublishTime"),
            article.get("pubDate"),
            article.get("publishedAt"),
            article.get("datePublished"),
        )
    )
    article_type = (
        _clean_text(_coalesce(article.get("type"), article.get("contentType"))).upper()
        or "STORY"
    )
    related_tickers = _extract_related_tickers(article, candidate, default_symbol=symbol)
    thumbnail = _extract_thumbnail(article, candidate)
    item_uuid = _extract_uuid(
        article,
        candidate,
        fallback_link=link,
        fallback_title=title,
        symbol=symbol,
    )

    return NewsItem(
        uuid=item_uuid,
        title=title,
        link=link,
        publisher=provider,
        provider_publish_time=publish_time,
        article_type=article_type,
        related_tickers=related_tickers,
        thumbnail=thumbnail,
        summary=summary,
    )


def _parse_json_ld_article(
    candidate: Mapping[str, Any], symbol: str, base_url: str
) -> NewsItem | None:
    normalized = {
        "title": candidate.get("headline"),
        "url": candidate.get("url"),
        "datePublished": candidate.get("datePublished"),
        "summary": candidate.get("description"),
        "publisher": (_as_mapping(candidate.get("publisher")) or {}).get("name"),
        "thumbnail": {
            "resolutions": [
                {"url": image}
                for image in _as_sequence(candidate.get("image")) or []
                if isinstance(image, str) and image.strip()
            ]
        },
    }
    return _normalize_article(normalized, symbol=symbol, base_url=base_url)


def _deduplicate_items(items: Sequence[NewsItem]) -> list[dict[str, Any]]:
    deduplicated: list[dict[str, Any]] = []
    seen: set[str] = set()
    for item in items:
        key = item.link or item.uuid
        if key in seen:
            continue
        seen.add(key)
        deduplicated.append(item.to_dict())
    return deduplicated


def _extract_embedded_articles(html: str, symbol: str, base_url: str) -> list[dict[str, Any]]:
    json_blob = _extract_json_object(html, ROOT_APP_MAIN_MARKER)
    if json_blob is None:
        return []

    try:
        parsed = json.loads(json_blob)
    except json.JSONDecodeError:
        LOGGER.debug("Failed to decode Yahoo root.App.main JSON", exc_info=True)
        return []

    stores = (
        _as_mapping(parsed.get("context"))
        and _as_mapping((_as_mapping(parsed.get("context")) or {}).get("dispatcher"))
        and _as_mapping(
            ((_as_mapping(parsed.get("context")) or {}).get("dispatcher") or {}).get("stores")
        )
    )
    search_root = stores or parsed

    items = []
    for mapping in _iter_mappings(search_root):
        if not _looks_like_article(mapping):
            continue
        item = _normalize_article(mapping, symbol=symbol, base_url=base_url)
        if item is not None:
            items.append(item)

    return _deduplicate_items(items)


def _extract_json_ld_articles(
    soup: BeautifulSoup, symbol: str, base_url: str
) -> list[dict[str, Any]]:
    items: list[NewsItem] = []

    def visit(candidate: Any) -> None:
        mapping = _as_mapping(candidate)
        if mapping is None:
            return

        candidate_type = _clean_text(mapping.get("@type")).lower()
        if candidate_type in {"newsarticle", "article"}:
            item = _parse_json_ld_article(
                mapping,
                symbol=symbol,
                base_url=base_url,
            )
            if item is not None:
                items.append(item)

        for key in ("itemListElement", "item"):
            nested = mapping.get(key)
            sequence = _as_sequence(nested)
            if sequence is not None:
                for entry in sequence:
                    visit(entry)
                continue
            visit(nested)

    for script in soup.find_all("script", attrs={"type": "application/ld+json"}):
        raw_text = script.string or script.get_text(strip=True)
        if not raw_text:
            continue
        try:
            payload = json.loads(raw_text)
        except json.JSONDecodeError:
            continue

        if isinstance(payload, list):
            for entry in payload:
                visit(entry)
        else:
            visit(payload)

    return _deduplicate_items(items)


def _parse_meta_text(section: Tag) -> tuple[str, float]:
    meta_node = section.select_one(
        "div.publishing, div[data-test='story-meta'], div[class*='publishing'], footer"
    )
    meta_text = meta_node.get_text(" ", strip=True) if meta_node else ""
    if not meta_text:
        return "Unknown", _utc_now().timestamp()

    parts = [part.strip() for part in re.split(r"[|•·]", meta_text) if part.strip()]
    if len(parts) >= 2:
        return parts[0], _parse_publish_time(parts[-1])

    parsed_time = _parse_relative_time(parts[0]) if parts else None
    if parsed_time is not None:
        return "Unknown", parsed_time.timestamp()
    return parts[0] if parts else "Unknown", _utc_now().timestamp()


def _extract_dom_related_tickers(section: Tag, symbol: str) -> list[str]:
    related: list[str] = []
    for node in section.select("a.ticker, a[data-symbol], [data-symbol]"):
        raw_symbol = node.get("data-symbol") or node.get("aria-label") or node.get("title")
        normalized = _clean_text(raw_symbol).upper()
        if normalized and normalized not in related:
            related.append(normalized)
    return related or [symbol.upper()]


def _extract_dom_thumbnail(section: Tag) -> dict[str, Any]:
    image = section.select_one("img[src], img[data-src], img[data-lazy-src]")
    if image is None:
        return {}
    url = _coalesce(image.get("src"), image.get("data-src"), image.get("data-lazy-src"))
    cleaned = _clean_text(url)
    if not cleaned:
        return {}
    return {"resolutions": [{"url": cleaned}]}


def _parse_dom_story(section: Tag, symbol: str, base_url: str) -> NewsItem | None:
    title_node = section.select_one("h3")
    if title_node is None:
        return None

    link_node = title_node.find_parent("a") or section.select_one("a[href]")
    if link_node is None:
        return None

    title = title_node.get_text(" ", strip=True)
    link = _normalize_url(link_node.get("href"), base_url)
    if not title or not link:
        return None

    publisher, publish_time = _parse_meta_text(section)
    summary_node = section.select_one("p")
    summary = summary_node.get_text(" ", strip=True) if summary_node is not None else ""
    uuid_value = _extract_uuid(
        {"data-ylk": link_node.get("data-ylk", "")},
        fallback_link=link,
        fallback_title=title,
        symbol=symbol,
    )

    return NewsItem(
        uuid=uuid_value,
        title=title,
        link=link,
        publisher=publisher,
        provider_publish_time=publish_time,
        article_type="STORY",
        related_tickers=_extract_dom_related_tickers(section, symbol),
        thumbnail=_extract_dom_thumbnail(section),
        summary=summary,
    )


def _extract_dom_articles(soup: BeautifulSoup, symbol: str, base_url: str) -> list[dict[str, Any]]:
    items: list[NewsItem] = []
    containers = soup.select(
        "section[data-testid='storyitem'], li.story-item, div[data-testid='storyitem']"
    )

    for container in containers:
        section = (
            container
            if container.name == "section"
            else container.select_one("section[data-testid='storyitem']") or container
        )
        item = _parse_dom_story(section, symbol=symbol, base_url=base_url)
        if item is not None:
            items.append(item)

    return _deduplicate_items(items)


def parse_news_html(
    html: str,
    symbol: str,
    base_url: str = DEFAULT_BASE_URL,
) -> list[dict[str, Any]]:
    soup = BeautifulSoup(html, "html.parser")

    for extractor in (
        lambda: _extract_embedded_articles(html, symbol=symbol, base_url=base_url),
        lambda: _extract_json_ld_articles(soup, symbol=symbol, base_url=base_url),
        lambda: _extract_dom_articles(soup, symbol=symbol, base_url=base_url),
    ):
        items = extractor()
        if items:
            return items

    return []


class YahooFinanceNewsScraper:
    """Fetch and parse Yahoo Finance quote/news pages."""

    def __init__(
        self,
        session: requests.Session | None = None,
        timeout: int = DEFAULT_TIMEOUT,
        base_urls: Sequence[str] | None = None,
    ) -> None:
        self.session = session or requests.Session()
        self.timeout = timeout
        self.base_urls = tuple(base_urls) if base_urls else ()

    def _candidate_base_urls(self, symbol: str) -> tuple[str, ...]:
        if self.base_urls:
            return tuple(self.base_urls)
        if symbol.upper().endswith(".DE"):
            return (GERMAN_BASE_URL, DEFAULT_BASE_URL)
        return (DEFAULT_BASE_URL, GERMAN_BASE_URL)

    @staticmethod
    def _build_url(base_url: str, symbol: str) -> str:
        return f"{base_url}/quote/{symbol}/news?p={symbol}"

    def _handle_consent(
        self,
        response: requests.Response,
        headers: Mapping[str, str],
    ) -> requests.Response:
        soup = BeautifulSoup(response.text, "html.parser")
        form = soup.select_one("form")
        if form is None:
            return response

        inputs = {
            field.get("name"): field.get("value", "")
            for field in form.select("input[name]")
            if field.get("name")
        }
        if "csrfToken" not in inputs:
            return response

        action = form.get("action") or response.url
        consent_url = urljoin(response.url, action)
        payload = dict(inputs)
        payload.setdefault("namespace", "yahoo")
        payload["agree"] = payload.get("agree") or "agree"

        consent_response = self.session.post(
            consent_url,
            data=payload,
            headers=dict(headers),
            timeout=self.timeout,
        )
        consent_response.raise_for_status()
        return self.session.get(response.url, headers=dict(headers), timeout=self.timeout)

    def fetch_news(self, symbol: str) -> list[dict[str, Any]]:
        last_error: Exception | None = None

        for base_url in self._candidate_base_urls(symbol):
            url = self._build_url(base_url, symbol)
            try:
                response = self.session.get(
                    url,
                    headers=dict(DEFAULT_HEADERS),
                    timeout=self.timeout,
                )
                response.raise_for_status()
                response = self._handle_consent(response, DEFAULT_HEADERS)
                response.raise_for_status()
            except requests.RequestException as exc:
                last_error = exc
                LOGGER.warning("Request failed for %s via %s: %s", symbol, base_url, exc)
                continue

            items = parse_news_html(response.text, symbol=symbol, base_url=base_url)
            if items:
                return items

            LOGGER.info("No news items parsed for %s via %s", symbol, base_url)

        if last_error is not None:
            LOGGER.error("Failed to fetch news for %s: %s", symbol, last_error)
        return []


class Ticker:
    """Ticker facade that preserves the original SimpleYNews API."""

    def __init__(
        self,
        symbol: str,
        *,
        session: requests.Session | None = None,
        timeout: int = DEFAULT_TIMEOUT,
        base_urls: Sequence[str] | None = None,
        scraper: YahooFinanceNewsScraper | None = None,
    ) -> None:
        self.symbol = symbol
        self._scraper = scraper or YahooFinanceNewsScraper(
            session=session,
            timeout=timeout,
            base_urls=base_urls,
        )
        self._news_cache: list[dict[str, Any]] | None = None

    @property
    def news(self) -> list[dict[str, Any]]:
        return self.get_news()

    def get_news(self, force_refresh: bool = False) -> list[dict[str, Any]]:
        if force_refresh or self._news_cache is None:
            items = self._scraper.fetch_news(self.symbol)
            if items or force_refresh:
                self._news_cache = items
            elif self._news_cache is None:
                return []
        return self._news_cache or []


class SimpleYNews:
    """Namespace-style compatibility wrapper."""

    @staticmethod
    # pylint: disable=invalid-name
    def Ticker(symbol: str, **kwargs: Any) -> Ticker:
        return Ticker(symbol, **kwargs)
