from datetime import timedelta
import json
from unittest.mock import AsyncMock, Mock
from urllib.parse import parse_qsl, urlsplit
from typing import Any

import pytest

import silkworm.response as response_module
from silkworm.http import HttpClient
from silkworm.engine import Engine
from silkworm.middlewares import (
    CloudflareCrawlMiddleware,
    DelayMiddleware,
    ProxyMiddleware,
    RequestResponseStreamMiddleware,
    RetryMiddleware,
)
from silkworm.request import Request
from silkworm.response import HTMLResponse, Response
from silkworm.spiders import Spider


class _StubResponse:
    def __init__(
        self, *, status: Any = 200, headers: Any = None, body: bytes | str = b""
    ) -> None:
        self.status = status
        self.headers = headers or {}
        self._body = body

    async def read(self) -> bytes:
        if isinstance(self._body, bytes):
            return self._body
        return str(self._body).encode("utf-8")

    async def text(self) -> str:
        if isinstance(self._body, bytes):
            return self._body.decode("utf-8", errors="replace")
        return str(self._body)


class _RecordingClient:
    def __init__(self) -> None:
        self.calls: list[tuple[Any, str, dict[str, Any]]] = []

    async def request(self, method: Any, url: str, **kwargs: Any) -> _StubResponse:
        self.calls.append((method, url, kwargs))
        return _StubResponse()


class _MethodStub:
    POST = "POST"


def test_request_replace_creates_new_request_with_updates():
    original = Request(
        url="https://example.com",
        method="GET",
        headers={"X-Token": "abc"},
        meta={"foo": 1},
    )

    updated = original.replace(method="POST", meta={"foo": 2})

    assert updated is not original
    assert updated.method == "POST"
    assert updated.meta["foo"] == 2
    assert original.method == "GET"
    assert original.meta["foo"] == 1


def test_response_follow_inherits_callback_and_joins_url():
    def callback(resp: Response) -> None:
        return None

    req = Request(url="http://example.com/dir/page", callback=callback)
    resp = Response(url=req.url, status=200, headers={}, body=b"", request=req)

    next_req = resp.follow("next")

    assert next_req.url == "http://example.com/dir/next"
    assert next_req.callback is callback


def test_htmlresponse_follow_works_with_slots():
    def callback(resp: Response) -> None:
        return None

    req = Request(url="http://example.com/dir/page", callback=callback)
    resp = HTMLResponse(url=req.url, status=200, headers={}, body=b"", request=req)

    next_req = resp.follow("next")

    assert next_req.url == "http://example.com/dir/next"
    assert next_req.callback is callback


def test_response_follow_all_joins_urls_and_uses_callback():
    def callback(resp: Response) -> None:
        return None

    req = Request(url="http://example.com/dir/page", callback=callback)
    resp = Response(url=req.url, status=200, headers={}, body=b"", request=req)

    next_reqs = resp.follow_all(["next", None, "../up"])

    assert [req.url for req in next_reqs] == [
        "http://example.com/dir/next",
        "http://example.com/up",
    ]
    assert all(req.callback is callback for req in next_reqs)


async def test_htmlresponse_css_aliases_select(monkeypatch: pytest.MonkeyPatch):
    req = Request(url="http://example.com")
    resp = HTMLResponse(
        url=req.url,
        status=200,
        headers={},
        body=b"<html></html>",
        request=req,
    )

    document = type("Document", (), {})()
    document.select = AsyncMock(return_value=["first", "second"])
    first_link = object()
    document.select_first = AsyncMock(return_value=first_link)
    parse_mock = AsyncMock(return_value=document)
    monkeypatch.setattr(response_module, "parse_async", parse_mock)

    links = await resp.css("a")
    first = await resp.css_first("a")

    assert len(links) == 2
    assert first is first_link
    parse_mock.assert_awaited_once_with(
        resp.text,
        max_size_bytes=resp.doc_max_size_bytes,
    )
    document.select.assert_awaited_once_with("a")
    document.select_first.assert_awaited_once_with("a")


async def test_htmlresponse_find_aliases_select_first(monkeypatch: pytest.MonkeyPatch):
    req = Request(url="http://example.com")
    resp = HTMLResponse(
        url=req.url,
        status=200,
        headers={},
        body=b"<html></html>",
        request=req,
    )

    document = type("Document", (), {})()
    found_link = object()
    document.select_first = AsyncMock(return_value=found_link)
    parse_mock = AsyncMock(return_value=document)
    monkeypatch.setattr(response_module, "parse_async", parse_mock)

    found = await resp.find("a")

    assert found is found_link
    parse_mock.assert_awaited_once_with(
        resp.text,
        max_size_bytes=resp.doc_max_size_bytes,
    )
    document.select_first.assert_awaited_once_with("a")


async def test_htmlresponse_prettify_uses_scraper_rs(monkeypatch: pytest.MonkeyPatch):
    req = Request(url="http://example.com")
    resp = HTMLResponse(
        url=req.url,
        status=200,
        headers={},
        body=b"<html></html>",
        request=req,
    )

    document = type("Document", (), {})()
    prettify_mock = AsyncMock(return_value="<html>\n  <body></body>\n</html>")
    document.prettify = prettify_mock
    parse_mock = AsyncMock(return_value=document)
    monkeypatch.setattr(response_module, "parse_async", parse_mock)

    pretty = await resp.prettify()

    assert pretty == "<html>\n  <body></body>\n</html>"
    parse_mock.assert_awaited_once_with(
        resp.text,
        max_size_bytes=resp.doc_max_size_bytes,
    )
    prettify_mock.assert_awaited_once_with()


async def test_htmlresponse_close_releases_cached_document(
    monkeypatch: pytest.MonkeyPatch,
):
    req = Request(url="http://example.com")
    resp = HTMLResponse(
        url=req.url,
        status=200,
        headers={},
        body=b"<html></html>",
        request=req,
    )

    document = type("Document", (), {})()
    document.select = AsyncMock(return_value=[])
    document.close = Mock()
    parse_mock = AsyncMock(return_value=document)
    monkeypatch.setattr(response_module, "parse_async", parse_mock)

    await resp.select("a")
    resp.close()

    document.close.assert_called_once_with()


def test_htmlresponse_url_join_resolves_relative_url():
    req = Request(url="http://example.com/dir/page")
    resp = HTMLResponse(url=req.url, status=200, headers={}, body=b"", request=req)

    resolved = resp.url_join("../other")

    assert resolved == "http://example.com/other"


def test_response_close_releases_payload():
    req = Request(url="http://example.com")
    resp = Response(
        url=req.url,
        status=200,
        headers={"Content-Type": "text/html"},
        body=b"abc",
        request=req,
    )

    resp.close()
    resp.close()  # idempotent

    assert resp.body == b""
    assert resp.headers == {}


def test_httpclient_build_url_merges_params_with_existing_query():
    client = HttpClient()
    req = Request(
        url="https://example.com/search?q=foo&lang=en",
        params={"page": 2, "q": "bar"},
    )

    built = client._build_url(req)
    parsed = dict(parse_qsl(urlsplit(built).query))

    assert parsed == {"q": "bar", "lang": "en", "page": "2"}


def test_httpclient_normalize_headers_handles_multiple_shapes():
    client = HttpClient()
    raw_headers = [
        b"Content-Type: text/html; charset=utf-8",
        ("X-Test", " value "),
        "X-RateLimit: 10",
        "InvalidHeaderWithoutColon",
    ]

    normalized = client._normalize_headers(raw_headers)

    assert normalized["content-type"] == "text/html; charset=utf-8"
    assert normalized["x-test"] == "value"
    assert normalized["x-ratelimit"] == "10"


async def test_httpclient_follows_redirects():
    class RedirectClient(_RecordingClient):
        async def request(self, method: Any, url: str, **kwargs: Any) -> _StubResponse:  # type: ignore[override]
            self.calls.append((method, url, kwargs))
            if url.startswith("http://example.com/start"):
                return _StubResponse(
                    status=302,
                    headers={"Location": "/next", "Content-Type": "text/plain"},
                )
            return _StubResponse(
                status=200, headers={"Content-Type": "text/plain"}, body=b"ok"
            )

    client = HttpClient()
    client._client = RedirectClient()  # type: ignore[assignment]

    resp = await client.fetch(
        Request(url="http://example.com/start", params={"foo": "1"})
    )

    assert resp.status == 200
    assert resp.url == "http://example.com/next"
    assert resp.body == b"ok"
    assert resp.request.url == "http://example.com/next"
    assert client._client.calls[0][1] == "http://example.com/start?foo=1"
    assert client._client.calls[1][1] == "http://example.com/next"


async def test_httpclient_converts_string_proxy_to_rnet_proxy():
    client = HttpClient()
    client._client = _RecordingClient()  # type: ignore[assignment]

    await client.fetch(
        Request(
            url="http://example.com",
            meta={"proxy": "http://proxy.example:8080"},
        )
    )

    proxy = client._client.calls[0][2]["proxy"]

    assert proxy.__class__.__name__ in {"_DummyProxy", "Proxy"}
    assert not isinstance(proxy, str)
    assert "proxy.example:8080" in str(proxy)


async def test_httpclient_detects_redirect_loops():
    from silkworm.exceptions import HttpError

    class LoopingClient(_RecordingClient):
        async def request(self, method: Any, url: str, **kwargs: Any) -> _StubResponse:  # type: ignore[override]
            self.calls.append((method, url, kwargs))
            return _StubResponse(
                status=302, headers={"Location": "/loop", "Content-Type": "text/plain"}
            )

    client = HttpClient(max_redirects=2)
    client._client = LoopingClient()  # type: ignore[assignment]

    with pytest.raises(HttpError):
        await client.fetch(Request(url="http://example.com/loop"))


async def test_httpclient_handles_unhashable_status_codes():
    class UnhashableStatus:
        __hash__ = None

        def __int__(self) -> int:
            return 302

    class RedirectClient(_RecordingClient):
        async def request(  # type: ignore[override]
            self, method: Any, url: str, **kwargs: Any
        ) -> _StubResponse:
            self.calls.append((method, url, kwargs))
            if len(self.calls) == 1:
                return _StubResponse(
                    status=UnhashableStatus(),
                    headers={"Location": "/next", "Content-Type": "text/plain"},
                )
            return _StubResponse(
                status=200, headers={"Content-Type": "text/plain"}, body=b"done"
            )

    client = HttpClient()
    redirect_client = RedirectClient()
    client._client = redirect_client  # type: ignore[assignment]

    resp = await client.fetch(Request(url="http://example.com/start"))

    assert resp.status == 200
    assert resp.url == "http://example.com/next"
    assert redirect_client.calls[0][1] == "http://example.com/start"
    assert redirect_client.calls[1][1] == "http://example.com/next"


async def test_httpclient_sets_keep_alive():
    client = HttpClient(keep_alive=True)
    recording = _RecordingClient()
    client._client = recording  # type: ignore[assignment]

    await client.fetch(Request(url="http://example.com/keep-alive"))

    assert recording.calls
    kwargs = recording.calls[0][2]
    assert kwargs.get("keep_alive") is True
    assert kwargs["headers"].get("Connection") == "keep-alive"


async def test_httpclient_keep_alive_when_kwarg_not_supported():
    class StrictClient:
        def __init__(self) -> None:
            self.calls: list[dict[str, Any]] = []

        async def request(
            self,
            method: Any,
            url: str,
            *,
            headers: dict[str, str],
            data: Any = None,
            json: Any = None,
            proxy: Any = None,
            timeout: Any = None,
        ) -> _StubResponse:
            self.calls.append(
                {
                    "method": method,
                    "url": url,
                    "headers": headers,
                    "data": data,
                    "json": json,
                    "proxy": proxy,
                    "timeout": timeout,
                }
            )
            return _StubResponse(headers={"Content-Type": "text/plain"})

    client = HttpClient(keep_alive=True)
    strict = StrictClient()
    client._client = strict  # type: ignore[assignment]

    resp = await client.fetch(Request(url="http://example.com/no-kw"))

    assert resp.status == 200
    assert strict.calls
    first_call = strict.calls[0]
    assert first_call["headers"].get("Connection") == "keep-alive"


async def test_httpclient_uses_timedelta_timeout():
    client = HttpClient()
    recording = _RecordingClient()
    client._client = recording  # type: ignore[assignment]

    resp = await client.fetch(Request(url="http://example.com", timeout=5))

    assert resp.status == 200
    assert len(recording.calls) == 1
    assert isinstance(recording.calls[0][2]["timeout"], timedelta)


async def test_httpclient_returns_synthetic_response_from_meta():
    client = HttpClient()
    request = Request(
        url="https://example.com/original",
        meta={
            "_silkworm_mock_response": {
                "url": "https://example.com/synthetic",
                "status": 202,
                "headers": {"Content-Type": "application/json"},
                "body": '{"ok": true}',
            }
        },
    )

    response = await client.fetch(request)

    assert response.url == "https://example.com/synthetic"
    assert response.status == 202
    assert response.text == '{"ok": true}'


async def test_retry_middleware_returns_retry_request(monkeypatch: pytest.MonkeyPatch):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    middleware = RetryMiddleware(max_times=2, backoff_base=0.1)
    request = Request(url="http://example.com")
    response = Response(
        url=request.url, status=500, headers={}, body=b"", request=request
    )

    result = await middleware.process_response(response, Spider())

    assert isinstance(result, Request)
    assert result is not request
    assert result.meta["retry_times"] == 1
    assert result.dont_filter is True
    assert sleep_calls == [0.1]


async def test_retry_middleware_sleep_codes_extend_retry(
    monkeypatch: pytest.MonkeyPatch,
):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    middleware = RetryMiddleware(max_times=1, sleep_http_codes=[403], backoff_base=0.2)
    request = Request(url="http://example.com")
    response = Response(
        url=request.url, status=403, headers={}, body=b"", request=request
    )

    result = await middleware.process_response(response, Spider())

    assert isinstance(result, Request)
    assert result.meta["retry_times"] == 1
    assert sleep_calls == [0.2]


async def test_retry_middleware_retry_without_sleep(monkeypatch: pytest.MonkeyPatch):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    middleware = RetryMiddleware(
        max_times=1, retry_http_codes=[500], sleep_http_codes=[]
    )
    request = Request(url="http://example.com")
    response = Response(
        url=request.url, status=500, headers={}, body=b"", request=request
    )

    result = await middleware.process_response(response, Spider())

    assert isinstance(result, Request)
    assert result.meta["retry_times"] == 1
    assert sleep_calls == []


async def test_engine_closes_responses(monkeypatch: pytest.MonkeyPatch):
    class DummySpider(Spider):
        name = "closer"

        async def start_requests(self):
            yield Request(url="http://example.com", callback=self.parse)

        async def parse(self, response: Response):
            return None

    class ClosableResponse(Response):
        __slots__ = ("closed",)

        def __init__(self, *args, **kwargs) -> None:
            super().__init__(*args, **kwargs)
            self.closed = False

        def close(self) -> None:
            self.closed = True

    spider = DummySpider()
    engine = Engine(spider, concurrency=1)
    seen: list[ClosableResponse] = []

    async def fake_fetch(req: Request) -> Response:
        resp = ClosableResponse(
            url=req.url, status=200, headers={}, body=b"", request=req
        )
        seen.append(resp)
        return resp

    engine.http.fetch = fake_fetch  # type: ignore[assignment]

    await engine.run()

    assert seen and all(r.closed for r in seen)


async def test_retry_middleware_stops_after_max_times(monkeypatch: pytest.MonkeyPatch):
    async def fake_sleep(_: float) -> None:
        return None

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)
    middleware = RetryMiddleware(max_times=1)
    request = Request(url="http://example.com", meta={"retry_times": 1})
    response = Response(
        url=request.url, status=500, headers={}, body=b"", request=request
    )

    result = await middleware.process_response(response, Spider())

    assert result is response


async def test_engine_retries_requests_even_if_url_seen(
    monkeypatch: pytest.MonkeyPatch,
):
    async def fake_sleep(_: float) -> None:
        return None

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    class DummySpider(Spider):
        name = "retryer"

        async def start_requests(self):
            yield Request(url="http://example.com", callback=self.parse)

        async def parse(self, response: Response):
            return None

    statuses = [429, 200]
    seen_requests: list[tuple[int, int]] = []

    engine = Engine(
        DummySpider(),
        concurrency=1,
        response_middlewares=[RetryMiddleware(max_times=1, backoff_base=0.0)],
    )

    async def fake_fetch(req: Request) -> Response:
        status = statuses.pop(0)
        retry_raw = req.meta.get("retry_times", 0)
        retry_times = int(retry_raw) if isinstance(retry_raw, (int, float, str)) else 0
        seen_requests.append((status, retry_times))
        return Response(url=req.url, status=status, headers={}, body=b"", request=req)

    engine.http.fetch = fake_fetch  # type: ignore[assignment]

    await engine.run()

    assert seen_requests == [(429, 0), (200, 1)]


async def test_delay_middleware_fixed_delay(monkeypatch: pytest.MonkeyPatch):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    middleware = DelayMiddleware(delay=1.5)
    request = Request(url="http://example.com")

    result = await middleware.process_request(request, Spider())

    assert result is request
    assert sleep_calls == [1.5]


async def test_delay_middleware_random_delay(monkeypatch: pytest.MonkeyPatch):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    middleware = DelayMiddleware(min_delay=0.5, max_delay=2.0)
    request = Request(url="http://example.com")

    result = await middleware.process_request(request, Spider())

    assert result is request
    assert len(sleep_calls) == 1
    assert 0.5 <= sleep_calls[0] <= 2.0


async def test_delay_middleware_custom_function(monkeypatch: pytest.MonkeyPatch):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    def custom_delay(request: Request, spider: Spider) -> float:
        return 3.0 if "slow" in request.url else 0.5

    middleware = DelayMiddleware(delay_func=custom_delay)

    slow_request = Request(url="http://slow.example.com")
    result1 = await middleware.process_request(slow_request, Spider())
    assert result1 is slow_request
    assert sleep_calls[-1] == 3.0

    fast_request = Request(url="http://fast.example.com")
    result2 = await middleware.process_request(fast_request, Spider())
    assert result2 is fast_request
    assert sleep_calls[-1] == 0.5


def test_delay_middleware_validation_errors():
    # Must provide at least one configuration
    with pytest.raises(ValueError, match="Must provide one of"):
        DelayMiddleware()

    # Cannot mix delay strategies
    with pytest.raises(ValueError, match="Cannot use both"):
        DelayMiddleware(delay=1.0, min_delay=0.5)

    with pytest.raises(ValueError, match="cannot be used with"):
        DelayMiddleware(delay=1.0, delay_func=lambda r, s: 1.0)

    # min_delay and max_delay must both be provided
    with pytest.raises(ValueError, match="Both min_delay and max_delay"):
        DelayMiddleware(min_delay=0.5)

    with pytest.raises(ValueError, match="Both min_delay and max_delay"):
        DelayMiddleware(max_delay=2.0)

    # Negative values not allowed
    with pytest.raises(ValueError, match="must be non-negative"):
        DelayMiddleware(delay=-1.0)

    with pytest.raises(ValueError, match="must be non-negative"):
        DelayMiddleware(min_delay=-0.5, max_delay=2.0)

    with pytest.raises(ValueError, match="must be non-negative"):
        DelayMiddleware(min_delay=0.5, max_delay=-2.0)

    # min_delay must be <= max_delay
    with pytest.raises(ValueError, match="must be less than or equal to"):
        DelayMiddleware(min_delay=2.0, max_delay=0.5)


async def test_delay_middleware_zero_delay(monkeypatch: pytest.MonkeyPatch):
    sleep_calls: list[float] = []

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    middleware = DelayMiddleware(delay=0.0)
    request = Request(url="http://example.com")

    result = await middleware.process_request(request, Spider())

    assert result is request
    # Zero delay should not call sleep
    assert sleep_calls == []


# ProxyMiddleware tests
async def test_proxy_middleware_round_robin_selection():
    from silkworm.middlewares import ProxyMiddleware

    proxies = ["http://proxy1:8080", "http://proxy2:8080", "http://proxy3:8080"]
    middleware = ProxyMiddleware(proxies=proxies)

    requests = [Request(url=f"http://example.com/{i}") for i in range(5)]
    results = [await middleware.process_request(req, Spider()) for req in requests]

    # Should cycle through proxies in order
    assert results[0].meta["proxy"] == "http://proxy1:8080"
    assert results[1].meta["proxy"] == "http://proxy2:8080"
    assert results[2].meta["proxy"] == "http://proxy3:8080"
    assert results[3].meta["proxy"] == "http://proxy1:8080"  # cycle back
    assert results[4].meta["proxy"] == "http://proxy2:8080"


async def test_proxy_middleware_random_selection():
    from silkworm.middlewares import ProxyMiddleware

    proxies = ["http://proxy1:8080", "http://proxy2:8080", "http://proxy3:8080"]
    middleware = ProxyMiddleware(proxies=proxies, random_selection=True)

    # Generate multiple requests to test random selection
    requests = [Request(url=f"http://example.com/{i}") for i in range(30)]
    results = [await middleware.process_request(req, Spider()) for req in requests]

    # All assigned proxies should be from our list
    assigned_proxies = [r.meta["proxy"] for r in results]
    assert all(p in proxies for p in assigned_proxies)

    # With random selection, we should see some variation
    # (statistically unlikely to get all the same proxy with 30 requests and 3 proxies)
    unique_proxies = set(assigned_proxies)
    assert len(unique_proxies) > 1


async def test_proxy_middleware_from_file(tmp_path):
    from silkworm.middlewares import ProxyMiddleware

    # Create a temporary proxy file
    proxy_file = tmp_path / "proxies.txt"
    proxy_file.write_text(
        "http://proxy1:8080\nhttp://proxy2:8080\n\nhttp://proxy3:8080\n"
    )

    middleware = ProxyMiddleware(proxy_file=proxy_file)

    assert len(middleware.proxies) == 3
    assert "http://proxy1:8080" in middleware.proxies
    assert "http://proxy2:8080" in middleware.proxies
    assert "http://proxy3:8080" in middleware.proxies

    # Test that it works
    request = Request(url="http://example.com")
    result = await middleware.process_request(request, Spider())
    assert result.meta["proxy"] in middleware.proxies


async def test_proxy_middleware_from_file_with_random_selection(tmp_path):
    # Create a temporary proxy file
    proxy_file = tmp_path / "proxies.txt"
    proxy_file.write_text(
        "http://proxy1:8080\nhttp://proxy2:8080\nhttp://proxy3:8080\n"
    )

    middleware = ProxyMiddleware(proxy_file=proxy_file, random_selection=True)

    # Generate multiple requests to test random selection
    requests = [Request(url=f"http://example.com/{i}") for i in range(30)]
    results = [await middleware.process_request(req, Spider()) for req in requests]

    # All assigned proxies should be from our list
    assigned_proxies = [r.meta["proxy"] for r in results]
    assert all(p in middleware.proxies for p in assigned_proxies)

    # With random selection, we should see some variation
    unique_proxies = set(assigned_proxies)
    assert len(unique_proxies) > 1


def test_proxy_middleware_validation_errors(tmp_path):
    # Must provide either proxies or proxy_file
    with pytest.raises(ValueError, match="Must provide either"):
        ProxyMiddleware()

    # Cannot provide both proxies and proxy_file
    proxy_file = tmp_path / "proxies.txt"
    proxy_file.write_text("http://proxy1:8080\n")

    with pytest.raises(ValueError, match="Cannot specify both"):
        ProxyMiddleware(proxies=["http://proxy1:8080"], proxy_file=proxy_file)

    # File must exist
    with pytest.raises(FileNotFoundError):
        ProxyMiddleware(proxy_file="nonexistent.txt")

    # File must contain at least one proxy
    empty_file = tmp_path / "empty.txt"
    empty_file.write_text("")

    with pytest.raises(ValueError, match="requires at least one proxy"):
        ProxyMiddleware(proxy_file=empty_file)

    # Empty proxies list should raise error
    with pytest.raises(ValueError, match="requires at least one proxy"):
        ProxyMiddleware(proxies=[])


async def test_proxy_middleware_rotates_proxy_after_exception():
    middleware = ProxyMiddleware(
        proxies=["http://proxy1:8080", "http://proxy2:8080", "http://proxy3:8080"]
    )
    request = await middleware.process_request(
        Request(url="http://example.com"), Spider()
    )

    retry_request = await middleware.process_exception(
        request,
        RuntimeError("proxy connection failed"),
        Spider(),
    )

    assert isinstance(retry_request, Request)
    assert retry_request.dont_filter is True
    assert retry_request.meta["proxy"] == "http://proxy2:8080"
    assert retry_request.meta["_proxy_failed_proxies"] == ["http://proxy1:8080"]
    assert retry_request.meta["_proxy_retry_times"] == 1


async def test_engine_retries_failed_request_with_another_proxy():
    class DummySpider(Spider):
        name = "proxy-retry"

        async def start_requests(self):
            yield Request(url="http://example.com", callback=self.parse)

        async def parse(self, response: Response):
            yield {"status": response.status}

    proxy_middleware = ProxyMiddleware(
        proxies=["http://proxy1:8080", "http://proxy2:8080", "http://proxy3:8080"]
    )
    engine = Engine(
        DummySpider(),
        concurrency=1,
        request_middlewares=[proxy_middleware],
    )

    seen_proxies: list[str] = []
    scraped_items: list[dict[str, int]] = []

    async def fake_process_item(item: Any) -> None:
        assert isinstance(item, dict)
        scraped_items.append(item)

    async def fake_fetch(req: Request) -> Response:
        proxy = req.meta.get("proxy")
        assert isinstance(proxy, str)
        seen_proxies.append(proxy)
        if len(seen_proxies) < 3:
            raise RuntimeError(f"Proxy failed for {req.url}")
        return Response(url=req.url, status=200, headers={}, body=b"ok", request=req)

    engine.http.fetch = fake_fetch  # type: ignore[assignment]
    engine._process_item = fake_process_item  # type: ignore[method-assign]

    await engine.run()

    assert seen_proxies == [
        "http://proxy1:8080",
        "http://proxy2:8080",
        "http://proxy3:8080",
    ]
    assert scraped_items == [{"status": 200}]


async def test_cloudflare_crawl_middleware_passthrough_without_meta():
    middleware = CloudflareCrawlMiddleware(account_id="acct", api_token="token")
    request = Request(url="https://example.com")

    result = await middleware.process_request(request, Spider())

    assert result is request


async def test_cloudflare_crawl_middleware_builds_synthetic_response(
    monkeypatch: pytest.MonkeyPatch,
):
    middleware = CloudflareCrawlMiddleware(
        account_id="acct",
        api_token="token",
        crawl_options={"limit": 5},
    )
    request = Request(
        url="https://example.com",
        meta={"cloudflare_crawl": {"depth": 2}},
    )

    async def fake_run_crawl(
        url: str,
        crawl_settings: dict[str, Any],
    ) -> dict[str, Any]:
        assert url == "https://example.com"
        assert crawl_settings == {"limit": 5, "depth": 2}
        return {
            "success": True,
            "result": {
                "status": "completed",
                "records": [{"url": "https://example.com", "markdown": "# Home"}],
            },
        }

    monkeypatch.setattr(middleware, "_run_crawl", fake_run_crawl)

    processed = await middleware.process_request(request, Spider())

    mock_response = processed.meta["_silkworm_mock_response"]
    assert isinstance(mock_response, dict)
    assert processed.meta["_cloudflare_crawl_applied"] is True
    assert mock_response["status"] == 200
    assert mock_response["headers"]["x-silkworm-source"] == "cloudflare-crawl"

    payload = json.loads(str(mock_response["body"]))
    assert payload["result"]["records"][0]["url"] == "https://example.com"


async def test_cloudflare_crawl_middleware_polls_until_completion(
    monkeypatch: pytest.MonkeyPatch,
):
    middleware = CloudflareCrawlMiddleware(
        account_id="acct",
        api_token="token",
        poll_interval=0.1,
        timeout=1.0,
    )
    sleep_calls: list[float] = []
    responses = iter(
        [
            {"success": True, "result": {"job_id": "job-1"}},
            {"success": True, "result": {"status": "running"}},
            {"success": True, "result": {"status": "completed"}},
            {
                "success": True,
                "result": {"status": "completed", "records": [{"url": "https://a"}]},
            },
        ]
    )

    api_paths: list[str] = []

    async def fake_api_request(method: str, path: str, **kwargs: Any) -> dict[str, Any]:
        api_paths.append(path)
        return next(responses)

    async def fake_sleep(delay: float) -> None:
        sleep_calls.append(delay)

    monkeypatch.setattr(middleware, "_api_request", fake_api_request)
    monkeypatch.setattr("silkworm.middlewares.asyncio.sleep", fake_sleep)

    payload = await middleware._run_crawl("https://example.com", {})

    assert payload["result"]["status"] == "completed"
    assert payload["result"]["records"][0]["url"] == "https://a"
    assert sleep_calls == [0.1]
    assert api_paths == [
        "/accounts/acct/browser-rendering/crawl",
        "/accounts/acct/browser-rendering/crawl/job-1?limit=1",
        "/accounts/acct/browser-rendering/crawl/job-1?limit=1",
        "/accounts/acct/browser-rendering/crawl/job-1",
    ]


async def test_cloudflare_crawl_middleware_runs_inside_engine(
    monkeypatch: pytest.MonkeyPatch,
):
    class CrawlSpider(Spider):
        name = "cloudflare-crawl"

        def __init__(self) -> None:
            super().__init__()
            self.payload: dict[str, Any] | None = None

        async def start_requests(self):
            yield Request(
                url="https://example.com",
                callback=self.parse,
                meta={"cloudflare_crawl": True},
            )

        async def parse(self, response: Response):
            self.payload = json.loads(response.text)
            return None

    spider = CrawlSpider()
    middleware = CloudflareCrawlMiddleware(account_id="acct", api_token="token")

    async def fake_run_crawl(
        url: str,
        crawl_settings: dict[str, Any],
    ) -> dict[str, Any]:
        assert url == "https://example.com"
        assert crawl_settings == {}
        return {
            "success": True,
            "result": {
                "status": "completed",
                "records": [{"url": "https://example.com/about"}],
            },
        }

    monkeypatch.setattr(middleware, "_run_crawl", fake_run_crawl)

    engine = Engine(spider, concurrency=1, request_middlewares=[middleware])
    await engine.run()

    assert spider.payload is not None
    assert spider.payload["result"]["records"][0]["url"] == "https://example.com/about"


async def test_request_response_stream_middleware_connects_events(
    monkeypatch: pytest.MonkeyPatch,
):
    client = _RecordingClient()
    monkeypatch.setattr("silkworm.middlewares.Client", lambda: client)
    monkeypatch.setattr("silkworm.middlewares.Method", _MethodStub)

    middleware = RequestResponseStreamMiddleware(
        "https://collector.example.com/events",
    )
    spider = Spider(name="stream-test")
    request = Request(
        url="https://example.com/api",
        method="POST",
        headers={"content-type": "application/json"},
        json={"hello": "world"},
        callback=spider.parse,
    )

    await middleware.open(spider)
    processed_request = await middleware.process_request(request, spider)
    response = Response(
        url=processed_request.url,
        status=201,
        headers={"content-type": "application/json"},
        body=b'{"ok": true}',
        request=processed_request,
    )
    await middleware.process_response(response, spider)
    await middleware.close(spider)

    assert len(client.calls) == 2

    request_event = client.calls[0][2]["json"]
    response_event = client.calls[1][2]["json"]

    assert request_event["event"] == "request"
    assert response_event["event"] == "response"
    assert request_event["exchange_id"] == response_event["exchange_id"]
    assert request_event["request"]["url"] == "https://example.com/api"
    assert request_event["request"]["body"]["kind"] == "json"
    assert request_event["request"]["body"]["value"] == {"hello": "world"}
    assert response_event["response"]["status"] == 201
    assert response_event["response"]["encoding"] == "ascii"
    assert isinstance(response_event["duration_ms"], float)


async def test_request_response_stream_middleware_batches_and_adds_auth_header(
    monkeypatch: pytest.MonkeyPatch,
):
    client = _RecordingClient()
    monkeypatch.setattr("silkworm.middlewares.Client", lambda: client)
    monkeypatch.setattr("silkworm.middlewares.Method", _MethodStub)

    middleware = RequestResponseStreamMiddleware(
        "https://collector.example.com/events",
        auth_token="secret-token",
        batch_size=2,
    )
    spider = Spider(name="stream-test")
    request = Request(url="https://example.com")

    await middleware.open(spider)
    processed_request = await middleware.process_request(request, spider)
    response = Response(
        url=processed_request.url,
        status=200,
        headers={"content-type": "text/plain"},
        body=b"ok",
        request=processed_request,
    )
    await middleware.process_response(response, spider)
    await middleware.close(spider)

    assert len(client.calls) == 1

    _, _, kwargs = client.calls[0]
    assert kwargs["headers"]["Authorization"] == "Bearer secret-token"
    payload = kwargs["json"]
    assert payload["count"] == 2
    assert len(payload["events"]) == 2
    assert payload["events"][0]["event"] == "request"
    assert payload["events"][1]["event"] == "response"


async def test_engine_runs_middleware_lifecycle_hooks():
    events: list[str] = []

    class LifecycleMiddleware:
        async def open(self, spider: Spider) -> None:
            events.append(f"open:{spider.name}")

        async def close(self, spider: Spider) -> None:
            events.append(f"close:{spider.name}")

        async def process_request(self, request: Request, spider: Spider) -> Request:
            events.append(f"request:{spider.name}")
            return request

    class LifecycleSpider(Spider):
        name = "lifecycle-spider"
        start_urls = ("https://example.com",)

        async def parse(self, response: Response):
            return None

    spider = LifecycleSpider()
    engine = Engine(
        spider,
        concurrency=1,
        request_middlewares=[LifecycleMiddleware()],
    )

    async def fake_fetch(request: Request) -> Response:
        return Response(
            url=request.url,
            status=200,
            headers={"content-type": "text/plain"},
            body=b"ok",
            request=request,
        )

    engine.http.fetch = fake_fetch  # type: ignore[method-assign]
    await engine.run()

    assert events == [
        "open:lifecycle-spider",
        "request:lifecycle-spider",
        "close:lifecycle-spider",
    ]


async def test_engine_shared_stream_middleware_only_opens_once(
    monkeypatch: pytest.MonkeyPatch,
):
    events: list[str] = []

    class SharedStreamMiddleware:
        async def open(self, spider: Spider) -> None:
            events.append(f"open:{spider.name}")

        async def close(self, spider: Spider) -> None:
            events.append(f"close:{spider.name}")

        async def process_request(self, request: Request, spider: Spider) -> Request:
            events.append("request")
            return request

        async def process_response(
            self,
            response: Response,
            spider: Spider,
        ) -> Response:
            events.append("response")
            return response

    class SharedSpider(Spider):
        name = "shared-stream-spider"
        start_urls = ("https://example.com",)

        async def parse(self, response: Response):
            return None

    spider = SharedSpider()
    middleware = SharedStreamMiddleware()
    engine = Engine(
        spider,
        concurrency=1,
        request_middlewares=[middleware],
        response_middlewares=[middleware],
    )

    async def fake_fetch(request: Request) -> Response:
        return Response(
            url=request.url,
            status=200,
            headers={"content-type": "text/plain"},
            body=b"ok",
            request=request,
        )

    engine.http.fetch = fake_fetch  # type: ignore[method-assign]
    await engine.run()

    assert events == [
        "open:shared-stream-spider",
        "request",
        "response",
        "close:shared-stream-spider",
    ]
