import warnings
import structlog
from urllib.robotparser import RobotFileParser
from typing import Callable
from httpx import URL
from .client import Plugin

log = structlog.get_logger("careful")


class RobotExclusionError(Exception):
    pass


def raise_robots_txt(url, robots):
    raise RobotExclusionError(f"{url} excluded by {robots.url}")


def warn_robots_txt(url, robots):
    warnings.warn(f"{url} excluded by {robots.url}")


RobotsRejectFunc = Callable[[str, RobotFileParser], None]


class RobotsTxtPlugin(Plugin):
    def __init__(
        self,
        as_user_agent: str | None = None,
        on_rejection: RobotsRejectFunc = raise_robots_txt,
    ):
        self._robots_for_domain = {}
        self._robots_ua = as_user_agent
        self._rejected_action = on_rejection

    def wrap(self, next_handler, client):
        def robots_handler(method, url, **kwargs):
            uurl = URL(url)
            domain = uurl.host
            if domain not in self._robots_for_domain:
                robots_url = f"{uurl.scheme}://{domain}/robots.txt"
                log.info("robots.txt request", url=robots_url)
                robots_resp = client._raw_request("GET", robots_url)
                # pass url for output, but don't do read
                parser = RobotFileParser(robots_url)
                log.debug("robots.txt response", response=robots_resp.text)
                parser.parse(robots_resp.text.splitlines())
                self._robots_for_domain[domain] = parser
            if not self._robots_for_domain[domain].can_fetch(
                self._robots_ua or client.headers["user-agent"], url
            ):
                self._rejected_action(url, self._robots_for_domain[domain])
            # if action doesn't raise an exception, the request goes through
            return next_handler(method, url, **kwargs)

        return robots_handler
