import pytest
import warnings
from careful.httpx import make_robots_txt_client, RobotExclusionError
from careful.httpx.robots import warn_robots_txt


def test_request_robots() -> None:
    client = make_robots_txt_client(as_user_agent="careful-httpx")

    with pytest.raises(RobotExclusionError):
        client.get("https://httpbin.org/deny")
    client.get("https://httpbin.org/allow")


def test_request_robots_warning() -> None:
    client = make_robots_txt_client(
        as_user_agent="careful-httpx",
        on_rejection=warn_robots_txt,
    )

    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        # should warn
        client.get("https://httpbin.org/deny")

        # check for warning
        assert len(w) == 1
        assert "robots.txt" in str(w[-1].message)
