"""HTTP fetch operations for web crawling.

This module provides operations for making HTTP requests (GET, POST)
and managing HTTP sessions in crawlers.
"""

from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse

import httpx
from banal import clean_dict, ensure_dict
from furl import furl

from memorious.helpers.forms import extract_form
from memorious.helpers.rule import parse_rule
from memorious.helpers.template import render_template
from memorious.operations import register
from memorious.util import make_url_key

if TYPE_CHECKING:
    from memorious.logic.context import Context


@register("fetch")
def fetch(context: Context, data: dict[str, Any]) -> None:
    """Fetch a URL via HTTP GET request.

    Performs an HTTP GET request on the URL specified in the data dict.
    Supports retry logic, URL rules filtering, incremental skipping,
    URL rewriting, pagination, and custom headers.

    Args:
        context: The crawler context.
        data: Must contain "url" key with the URL to fetch.

    Params:
        rules (optional): URL/content filtering rules. Default: match_all.
        retry (optional): Number of retry attempts. Default: 3.
        emit_errors (optional): If True, emit data even on HTTP errors. Default: False.
        headers (optional): Extra HTTP headers to send.
        base_url (optional): Base URL for resolving relative URLs.
        rewrite (optional): URL rewriting configuration with "method" and "data" keys.
            Methods: "template" (Jinja2), "replace" (string replace).
        pagination (optional): Pagination config with "param" key for page number.

    Example:
        ```yaml
        pipeline:
          # Simple fetch
          fetch:
            method: fetch
            params:
              rules:
                domain: example.com
              retry: 5
            handle:
              pass: parse

          # Fetch with URL rewriting and headers
          fetch_detail:
            method: fetch
            params:
              headers:
                Referer: https://example.com/search
              rewrite:
                method: template
                data: "https://example.com/doc/{{ doc_id }}"
            handle:
              pass: parse

          # Fetch with pagination
          fetch_list:
            method: fetch
            params:
              url: https://example.com/results
              pagination:
                param: page
            handle:
              pass: parse
        ```
    """
    # Apply extra headers
    _apply_headers(context)

    # Get URL from params or data
    url = _get_url(context, data)
    if url is None:
        context.log.warning("No URL for GET request")
        return

    # Apply pagination
    if "pagination" in context.params:
        pagination = ensure_dict(context.params["pagination"])
        if "param" in pagination:
            page = data.get("page", 1)
            f = furl(url)
            f.args[pagination["param"]] = page
            url = f.url

    # Apply URL rewriting
    if "rewrite" in context.params:
        rewrite = context.params["rewrite"]
        method = rewrite.get("method")
        method_data = rewrite.get("data")
        if method == "replace":
            url = url.replace(*method_data)
        elif method == "template":
            url = render_template(method_data, data)

    if url is None:
        context.log.error("No URL specified")
        return

    # Handle relative URLs
    f = furl(url)
    if f.scheme is None:
        base_url = context.params.get("base_url")
        if base_url:
            url = furl(base_url).join(f).url
        elif "url" in data:
            url = furl(data["url"]).join(f).url

    # Validate URL scheme
    if urlparse(url).scheme not in ("http", "https", ""):
        context.log.info("Fetch skipped, unsupported scheme", url=url)
        return

    attempt = data.pop("retry_attempt", 1)

    # Extract partial download state from previous attempt (if any)
    partial_path_str = data.pop("_partial_path", None)
    partial_bytes = data.pop("_partial_bytes", 0)
    partial_path = Path(partial_path_str) if partial_path_str else None

    result = None
    try:
        result = context.http.get(
            url,
            lazy=True,
            partial_path=partial_path,
            partial_bytes=partial_bytes,
        )
        rules = context.get("rules", {"match_all": {}})
        if not parse_rule(rules).apply(result):
            context.log.info("Fetch skip (rule)", url=result.url)
            # Cleanup partial file if skipping
            if partial_path and partial_path.exists():
                partial_path.unlink(missing_ok=True)
            return

        if not result.ok:
            context.emit_warning(
                "Fetch fail", url=result.url, status=result.status_code
            )
            # Cleanup partial file on HTTP error status
            if partial_path and partial_path.exists():
                partial_path.unlink(missing_ok=True)
            if not context.params.get("emit_errors", False):
                return
        else:
            context.log.info("Fetched", url=result.url, status=result.status_code)

        data.update(result.serialize())
        if url != result.url:
            tag = context.make_key(context.run_id, make_url_key(url), prefix="runs")
            context.set_tag(tag)
        context.emit(data=data)
    except httpx.HTTPError as ce:
        retries = int(context.get("retry", 3))
        if retries >= attempt:
            context.log.warning("Retry", url=url, error=str(ce), attempt=attempt)
            data["retry_attempt"] = attempt + 1

            if result is not None:
                # Pass partial download state for resume on retry
                if result.partial_path and result.partial_path.exists():
                    data["_partial_path"] = str(result.partial_path)
                    data["_partial_bytes"] = result.partial_bytes
                    context.log.info(
                        "Resumable download state saved",
                        url=url,
                        bytes_received=result.partial_bytes,
                    )

            context.recurse(data=data, delay=2**attempt)
        else:
            if result is not None:
                # Cleanup partial file on final failure
                if result.partial_path and result.partial_path.exists():
                    result.partial_path.unlink(missing_ok=True)
            context.emit_warning("Fetch fail", url=url, error=str(ce))


@register("session")
def session(context: Context, data: dict[str, Any]) -> None:
    """Configure HTTP session parameters for subsequent requests.

    Sets up authentication, user agent, referer, and proxy settings
    that will be used for all subsequent HTTP requests in this crawler run.

    Args:
        context: The crawler context.
        data: Passed through to next stage.

    Params:
        user (optional): Username for HTTP basic authentication.
        password (optional): Password for HTTP basic authentication.
        user_agent (optional): Custom User-Agent header.
        url (optional): URL to set as Referer header.
        proxy (optional): Proxy URL for HTTP/HTTPS requests.

    Example:
        ```yaml
        pipeline:
          setup_session:
            method: session
            params:
              user: "${HTTP_USER}"
              password: "${HTTP_PASSWORD}"
              user_agent: "MyBot/1.0"
            handle:
              pass: fetch
        ```
    """
    context.http.reset()

    user = context.get("user")
    password = context.get("password")

    if user is not None and password is not None:
        context.http.client.auth = (user, password)

    user_agent = context.get("user_agent")
    if user_agent is not None:
        context.http.client.headers["User-Agent"] = user_agent

    referer = context.get("url")
    if referer is not None:
        context.http.client.headers["Referer"] = referer

    proxy = context.get("proxy")
    if proxy is not None:
        context.http.client._mounts = {
            "http://": httpx.HTTPTransport(proxy=proxy),
            "https://": httpx.HTTPTransport(proxy=proxy),
        }

    # Explicitly save the session because no actual HTTP requests were made.
    context.http.save()
    context.emit(data=data)


def _get_url(context: Context, data: dict[str, Any]) -> str | None:
    """Get URL from context params or data, with optional templating."""
    url = context.params.get("url")
    if url is not None:
        return url.format(**data)
    return data.get("url")


def _get_headers(context: Context) -> dict[str, str]:
    """Get extra headers from context params."""
    return ensure_dict(context.params.get("headers"))


def _apply_headers(context: Context) -> None:
    """Apply extra headers to the HTTP client."""
    for key, value in _get_headers(context).items():
        context.http.client.headers[key] = value


def _get_post_data(context: Context, data: dict[str, Any]) -> dict[str, Any]:
    """Build POST data from params and data dict."""
    post_data = ensure_dict(context.params.get("data"))
    for post_key, data_key in ensure_dict(context.params.get("use_data")).items():
        if data_key in data:
            post_data[post_key] = data[data_key]
    return clean_dict(post_data)


@register("post")
def post(context: Context, data: dict[str, Any]) -> None:
    """Perform HTTP POST request with form data.

    Sends a POST request with form-urlencoded data to the specified URL.

    Args:
        context: The crawler context.
        data: Current stage data.

    Params:
        url (optional): Target URL. Default: data["url"].
        data (optional): Dictionary of form fields to POST.
        use_data (optional): Map of {post_field: data_key} to include from data dict.
        headers (optional): Extra HTTP headers.

    Example:
        ```yaml
        pipeline:
          submit_form:
            method: post
            params:
              url: https://example.com/search
              data:
                query: "test"
                page: 1
              use_data:
                session_id: sid
              headers:
                X-Custom-Header: value
            handle:
              pass: parse
        ```
    """
    _apply_headers(context)
    url = _get_url(context, data)
    if url is None:
        context.log.warning("No URL for POST request")
        return

    post_data = _get_post_data(context, data)
    context.log.debug("POST request", url=url)
    result = context.http.post(url, data=post_data)
    context.emit(data={**data, **result.serialize()})


@register("post_json")
def post_json(context: Context, data: dict[str, Any]) -> None:
    """Perform HTTP POST request with JSON body.

    Sends a POST request with a JSON payload to the specified URL.

    Args:
        context: The crawler context.
        data: Current stage data.

    Params:
        url (optional): Target URL. Default: data["url"].
        data (optional): Dictionary to send as JSON body.
        use_data (optional): Map of {json_field: data_key} to include from data dict.
        headers (optional): Extra HTTP headers.

    Example:
        ```yaml
        pipeline:
          api_call:
            method: post_json
            params:
              url: https://api.example.com/documents
              data:
                action: "search"
                limit: 100
              use_data:
                document_id: doc_id
            handle:
              pass: process
        ```
    """
    _apply_headers(context)
    url = _get_url(context, data)
    if url is None:
        context.log.warning("No URL for POST request")
        return

    json_data = _get_post_data(context, data)
    context.log.debug("POST JSON request", url=url)
    result = context.http.post(url, json_data=json_data)
    context.emit(data={**data, **result.serialize()})


@register("post_form")
def post_form(context: Context, data: dict[str, Any]) -> None:
    """Perform HTTP POST to an HTML form with its current values.

    Extracts form fields from an HTML page and submits them with
    optional additional data.

    Args:
        context: The crawler context.
        data: Current stage data (must have cached HTML response).

    Params:
        form (optional): XPath to locate the form element.
        data (optional): Additional form fields to add/override.
        use_data (optional): Map of {form_field: data_key} to include from data dict.
        headers (optional): Extra HTTP headers.

    Example:
        ```yaml
        pipeline:
          submit_search:
            method: post_form
            params:
              form: './/form[@id="search-form"]'
              data:
                query: "documents"
              use_data:
                csrf_token: token
            handle:
              pass: parse_results
        ```
    """
    _apply_headers(context)
    form_xpath = context.params.get("form")
    if not form_xpath:
        context.log.error("No form XPath specified")
        return

    result = context.http.rehash(data)
    if result.html is None:
        context.log.error("No HTML content to extract form from")
        return

    action, form_data = extract_form(result.html, form_xpath)
    if action is None:
        context.log.error("Form not found", xpath=form_xpath)
        return

    base_url = data.get("url", "")
    url = furl(base_url).join(action).url

    # Merge form data with additional data from params
    form_data.update(_get_post_data(context, data))
    context.log.debug("POST form request", url=url)
    post_result = context.http.post(url, data=form_data)
    context.emit(data={**data, **post_result.serialize()})
