"""
scrylo/scrapers/page_fetcher.py
--------------------------------
Upgraded page fetcher: Patchright → httpx fallback → LLM parsing.

Three-layer fetch strategy per URL:

  Layer 1 — Patchright (undetected Chromium)
    Handles: Cloudflare, JS-rendered content, cookie popups
    Used when: Patchright is installed
    Falls back to Layer 2 on any error

  Layer 2 — httpx (fast, lightweight)
    Handles: plain HTML, no bot protection
    Used when: Patchright not installed, or Layer 1 fails

  Layer 3 — LLM parsing (Groq → Phi-3)
    Triggered when: regex extraction found nothing useful on the page
    Passes cleaned page text to LLM with extraction prompt
    Returns structured JSON: [{full_name, job_title, company, email, ...}]

This module replaces the old page_fetcher.py entirely.
Public API is backward-compatible — fetch_pages() still works the same way.

New exports:
  fetch_pages(domains)             → {domain: html}       (unchanged)
  fetch_and_parse(url, lead_name)  → list[ParsedLead]     (new — for enrichment)
  stream_fetch_pages(domains)      → async gen (unchanged)
"""

from __future__ import annotations

import asyncio
import json
import logging
import random
import re
from typing import AsyncGenerator, Optional

import httpx

from scrylo.scrapers.browser import browser_fetch, patchright_available
from scrylo.scrapers.parser import ParsedLead

log = logging.getLogger("scrylo.fetcher")

_TIMEOUT = httpx.Timeout(10.0, connect=5.0)

_USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_2) AppleWebKit/605.1.15 "
    "(KHTML, like Gecko) Version/17.2 Safari/605.1.15",
]

_HEADERS_BASE = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-GB,en;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "DNT": "1",
    "Connection": "keep-alive",
}

# Company pages most likely to have team/email content
_ENRICHMENT_PATHS = [
    "",              # homepage
    "/about",
    "/team",
    "/about-us",
    "/contact",
    "/contact-us",
    "/people",
    "/leadership",
    "/company",
    "/impressum",    # GDPR-required contact page — Germany/EU goldmine
    "/imprint",      # English variant of impressum
    "/legal",
]


def _make_headers() -> dict:
    return {**_HEADERS_BASE, "User-Agent": random.choice(_USER_AGENTS)}


# ─────────────────────────────────────────────────────────────────────────────
# Core fetch — Patchright first, httpx fallback
# ─────────────────────────────────────────────────────────────────────────────

async def fetch_url(url: str) -> Optional[str]:
    """
    Fetch a single URL. Tries Patchright first, falls back to httpx.
    Returns rendered HTML or None.
    """
    # Try Patchright for JS-heavy/protected pages
    if patchright_available():
        html = await browser_fetch(url)
        if html and len(html) > 500:
            log.debug("Patchright: %s (%d chars)", url, len(html))
            return html
        # Patchright got nothing useful — fall through to httpx

    # httpx fallback — fast, reliable for plain HTML
    async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
        for attempt_url in _url_variants(url):
            try:
                r = await client.get(
                    attempt_url,
                    headers=_make_headers(),
                    follow_redirects=True,
                )
                if r.status_code == 200 and len(r.text) > 200:
                    log.debug("httpx: %s (%d chars)", attempt_url, len(r.text))
                    return r.text
            except Exception:
                continue

    return None


def _url_variants(url: str) -> list[str]:
    """Generate https/http variants for a URL."""
    url = url.rstrip("/")
    if url.startswith("http"):
        return [url]
    return [f"https://{url}", f"http://{url}",
            f"https://www.{url}", f"http://www.{url}"]


# ─────────────────────────────────────────────────────────────────────────────
# Domain enrichment — scan multiple paths for team/email data
# ─────────────────────────────────────────────────────────────────────────────

async def fetch_domain_pages(domain: str, max_pages: int = 6) -> str:
    """
    Fetch the most email-rich pages of a domain and concatenate HTML.
    Tries paths in order, stops early when enough content is found.
    Returns combined HTML string (empty if all fail).
    """
    domain = domain.lstrip("https://").lstrip("http://").strip("/")
    base = f"https://{domain}"

    combined = ""
    fetched  = 0

    for path in _ENRICHMENT_PATHS[:max_pages]:
        url = f"{base}{path}"
        html = await fetch_url(url)
        if html:
            combined += html
            fetched  += 1
            if len(combined) > 150_000:  # enough content, stop early
                break
        await asyncio.sleep(random.uniform(0.3, 0.8))  # polite delay

    log.debug("domain %s: fetched %d pages, %d chars total",
              domain, fetched, len(combined))
    return combined


# ─────────────────────────────────────────────────────────────────────────────
# LLM parsing pass — primary on structured pages, fallback on others
# ─────────────────────────────────────────────────────────────────────────────

def _strip_html(html: str) -> str:
    """
    Strip HTML tags and collapse whitespace.
    Returns clean plain text for LLM consumption.
    """
    # Remove script and style blocks entirely
    html = re.sub(r"<(script|style)[^>]*>.*?</(script|style)>",
                  "", html, flags=re.DOTALL | re.IGNORECASE)
    # Remove remaining tags
    text = re.sub(r"<[^>]+>", " ", html)
    # Collapse whitespace
    text = re.sub(r"\s+", " ", text).strip()
    return text


async def llm_parse_page(html: str, context: str = "") -> list[dict]:
    """
    Pass page text through the LLM to extract structured lead data.
    Returns list of dicts: [{full_name, job_title, company, email, ...}]
    Returns [] if LLM unavailable or finds nothing.

    context: optional hint e.g. "company page for Acme Corp" — improves accuracy
    """
    try:
        from scrylo.core.llm_adapter import parse_page_for_leads
    except ImportError:
        return []

    text = _strip_html(html)
    if not text or len(text) < 100:
        return []

    if context:
        text = f"Page context: {context}\n\n{text}"

    result = await asyncio.get_event_loop().run_in_executor(
        None, parse_page_for_leads, text
    )

    if not result.ok:
        return []

    try:
        parsed = json.loads(result.text)
        if isinstance(parsed, list):
            log.info("LLM parsed %d people from page (via %s)",
                     len(parsed), result.provider)
            return parsed
    except Exception:
        pass

    return []


# URL path segments that indicate a structured company page
# worth sending to the LLM regardless of regex results
_LLM_PRIORITY_PATHS = {
    "/team", "/about", "/about-us", "/people", "/leadership",
    "/company", "/impressum", "/imprint", "/meet-the-team",
    "/our-team", "/founders", "/management", "/staff",
}


def _is_structured_page(url: str) -> bool:
    """
    Return True if this URL is a structured company page where
    LLM parsing should run first (Option B).

    These pages have multiple people in predictable layouts.
    Regex often returns only the first person or misses emails.
    LLM reads the whole page and returns everyone.
    """
    path = "/" + url.split("/", 3)[-1].lower().split("?")[0]
    # Exact path match
    if path in _LLM_PRIORITY_PATHS:
        return True
    # Partial match — e.g. /about-us/team, /company/founders
    return any(seg in path for seg in _LLM_PRIORITY_PATHS)


def _people_from_llm(raw: list[dict], company_name: str, url: str) -> list[ParsedLead]:
    """Convert LLM JSON output to ParsedLead list."""
    leads = []
    for p in raw:
        name = (p.get("full_name") or "").strip()
        if not name:
            continue
        leads.append(ParsedLead(
            name=name,
            title=p.get("job_title", ""),
            company=p.get("company", "") or company_name,
            email=p.get("email", ""),
            location=p.get("location", ""),
            linkedin_url=p.get("linkedin_url", ""),
            domain=_domain_from_url(url),
        ))
    return leads


def _people_from_regex(page_data: dict, company_name: str, url: str) -> list[ParsedLead]:
    """Convert regex parse_company_page output to ParsedLead list."""
    leads = []
    for p in page_data.get("people", []):
        name = (p.get("name") or "").strip()
        if not name:
            continue
        leads.append(ParsedLead(
            name=name,
            title=p.get("title", ""),
            company=p.get("company", "") or company_name,
            email=p.get("email", ""),
            domain=_domain_from_url(url),
        ))
    return leads


async def fetch_and_parse(
    url: str,
    company_name: str = "",
    use_llm: bool = True,
) -> list[ParsedLead]:
    """
    Fetch a URL and extract structured lead data.

    Option B strategy:
      Structured company pages (/team /about /impressum etc.)
        → LLM first (reads everyone on page, handles any layout)
        → regex fallback (if no LLM key or LLM returns nothing)

      All other pages (homepage, search results, articles)
        → regex first (fast, free, handles simple HTML)
        → LLM fallback (if regex found nothing)

    This ensures LLM is used where it matters most — company team pages —
    without burning quota on every DDG result snippet.
    """
    from scrylo.scrapers.parser import parse_company_page

    html = await fetch_url(url)
    if not html:
        return []

    context = f"company page for {company_name}" if company_name else url
    is_structured = _is_structured_page(url)

    if use_llm and is_structured:
        # ── Structured page: LLM first ────────────────────────────────────
        # /team /about /impressum etc. — LLM reads everyone, any layout
        log.debug("LLM-first parse: %s", url)
        people = await llm_parse_page(html, context)
        if people:
            leads = _people_from_llm(people, company_name, url)
            if leads:
                log.info("LLM extracted %d people from %s", len(leads), url)
                return leads

        # LLM found nothing or unavailable — regex fallback
        try:
            page_data = parse_company_page(html, url)
            leads = _people_from_regex(page_data, company_name, url)
            if leads:
                log.debug("regex fallback extracted %d people from %s",
                          len(leads), url)
                return leads
        except Exception:
            pass

    else:
        # ── Other pages: regex first (fast, free) ─────────────────────────
        # Homepage, articles, search results — regex is sufficient
        try:
            page_data = parse_company_page(html, url)
            leads = _people_from_regex(page_data, company_name, url)
            if leads:
                return leads
        except Exception:
            pass

        # Regex found nothing — LLM fallback
        if use_llm:
            log.debug("regex found nothing, trying LLM: %s", url)
            people = await llm_parse_page(html, context)
            if people:
                return _people_from_llm(people, company_name, url)

    return []


def _domain_from_url(url: str) -> str:
    import re
    d = re.sub(r"^https?://(www\.)?", "", url).split("/")[0].lower()
    return d if "." in d else ""


# ─────────────────────────────────────────────────────────────────────────────
# Backward-compatible public API (used by run.py — unchanged signatures)
# ─────────────────────────────────────────────────────────────────────────────

async def _fetch_one(
    client: httpx.AsyncClient, domain: str
) -> tuple[str, Optional[str]]:
    """Fetch a single domain's homepage. Returns (domain, html|None)."""
    html = await fetch_url(f"https://{domain}")
    return domain, html


async def fetch_pages(
    domains: list[str],
    max_parallel: int = 6,
    delay_range: tuple[float, float] = (0.5, 1.5),
) -> dict[str, Optional[str]]:
    """
    Fetch homepage HTML for a list of domains.
    Returns {domain: html_or_None}.

    Patchright runs one page at a time (browser constraint).
    httpx runs up to max_parallel in parallel.
    """
    results: dict[str, Optional[str]] = {}

    if patchright_available():
        # Browser: sequential with delay (one page at a time per browser)
        for domain in domains:
            html = await fetch_url(f"https://{domain}")
            results[domain] = html
            await asyncio.sleep(random.uniform(*delay_range))
    else:
        # httpx: parallel batches
        semaphore = asyncio.Semaphore(max_parallel)

        async def _guarded(domain: str):
            async with semaphore:
                await asyncio.sleep(random.uniform(*delay_range))
                _, html = await _fetch_one(None, domain)  # type: ignore
                results[domain] = html

        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
            async def _guarded_client(domain: str):
                async with semaphore:
                    await asyncio.sleep(random.uniform(*delay_range))
                    domain_out, html = await _fetch_one(client, domain)
                    results[domain_out] = html

            tasks = [_guarded_client(d) for d in domains]
            await asyncio.gather(*tasks, return_exceptions=True)

    return results


async def stream_fetch_pages(
    domains: list[str],
    max_parallel: int = 6,
    delay_range: tuple[float, float] = (0.5, 1.5),
) -> AsyncGenerator[tuple[str, Optional[str]], None]:
    """
    Same as fetch_pages but yields (domain, html) as each completes.
    Used by Web UI SSE endpoint for live progress.
    """
    if patchright_available():
        for domain in domains:
            html = await fetch_url(f"https://{domain}")
            yield domain, html
            await asyncio.sleep(random.uniform(*delay_range))
    else:
        semaphore = asyncio.Semaphore(max_parallel)
        queue: asyncio.Queue[tuple[str, Optional[str]]] = asyncio.Queue()

        async def _worker(client: httpx.AsyncClient, domain: str):
            async with semaphore:
                await asyncio.sleep(random.uniform(*delay_range))
                result = await _fetch_one(client, domain)
                await queue.put(result)

        async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
            tasks = [asyncio.create_task(_worker(client, d)) for d in domains]
            for _ in range(len(domains)):
                yield await queue.get()
            await asyncio.gather(*tasks, return_exceptions=True)
