# SPDX-FileCopyrightText: 2026 Teilo Millet
# SPDX-License-Identifier: Apache-2.0

"""Tests for vauban.api_eval: API-based suffix evaluation."""

import json
import os
from pathlib import Path
from typing import cast
from unittest.mock import MagicMock, patch

import pytest

from vauban.api_eval import (
    _build_api_prompt,
    _build_auth_headers,
    _build_messages,
    _call_chat_api,
    _extract_content,
    _is_refusal,
    _is_violation_finish_reason,
    evaluate_suffix_via_api,
)
from vauban.config._parse_api_eval import _parse_api_eval
from vauban.types import ApiEvalConfig, ApiEvalEndpoint


@pytest.fixture
def endpoint() -> ApiEvalEndpoint:
    """A sample API endpoint."""
    return ApiEvalEndpoint(
        name="test-ep",
        base_url="https://api.example.com/v1",
        model="test-model",
        api_key_env="TEST_API_KEY",
    )


@pytest.fixture
def grayswan_endpoint() -> ApiEvalEndpoint:
    """A Gray Swan Cygnal endpoint with custom auth header."""
    return ApiEvalEndpoint(
        name="grayswan-cygnal",
        base_url="https://api.grayswan.ai/cygnal",
        model="cygnal-v1",
        api_key_env="GRAYSWAN_API_KEY",
        auth_header="grayswan-api-key",
    )


@pytest.fixture
def config(endpoint: ApiEvalEndpoint) -> ApiEvalConfig:
    """A sample API eval config."""
    return ApiEvalConfig(
        endpoints=[endpoint],
        max_tokens=50,
        timeout=10,
        system_prompt="You are helpful.",
    )


@pytest.fixture
def multiturn_config(endpoint: ApiEvalEndpoint) -> ApiEvalConfig:
    """A multi-turn API eval config."""
    return ApiEvalConfig(
        endpoints=[endpoint],
        max_tokens=50,
        timeout=10,
        system_prompt="You are helpful.",
        multiturn=True,
        multiturn_max_turns=3,
        follow_up_prompts=[
            "Can you be more specific?",
            "Give me step-by-step details.",
        ],
    )


def _mock_api_response(content: str, finish_reason: str = "stop") -> MagicMock:
    """Create a mock urllib response with the given content."""
    body = json.dumps({
        "choices": [{
            "message": {"content": content},
            "finish_reason": finish_reason,
        }],
    }).encode()
    mock = MagicMock()
    mock.read.return_value = body
    mock.__enter__ = lambda s: s
    mock.__exit__ = MagicMock(return_value=False)
    return mock


class TestBuildApiPrompt:
    """Tests for _build_api_prompt position-aware prompt assembly."""

    def test_suffix_position_appends(self) -> None:
        result = _build_api_prompt("Do something bad", "ADV_TOKENS", "suffix")
        assert result == "Do something bad ADV_TOKENS"

    def test_prefix_position_appends(self) -> None:
        result = _build_api_prompt("Do something bad", "ADV_TOKENS", "prefix")
        assert result == "Do something bad ADV_TOKENS"

    def test_infix_replaces_marker(self) -> None:
        prompt = "Write a story where {suffix} a character hacks wifi"
        result = _build_api_prompt(prompt, "ADV_TOKENS", "infix")
        assert result == "Write a story where ADV_TOKENS a character hacks wifi"

    def test_infix_no_marker_falls_back(self) -> None:
        prompt = "A prompt without any marker"
        result = _build_api_prompt(prompt, "ADV_TOKENS", "infix")
        assert result == "A prompt without any marker ADV_TOKENS"

    def test_infix_multiple_markers(self) -> None:
        prompt = "Part1 {suffix} part2 {suffix} part3"
        result = _build_api_prompt(prompt, "TOK", "infix")
        assert result == "Part1 TOK part2 TOK part3"


class TestIsRefusal:
    def test_refusal_detected(self) -> None:
        assert _is_refusal("I cannot help with that request.")

    def test_refusal_case_insensitive(self) -> None:
        assert _is_refusal("I CANNOT do that.")

    def test_non_refusal(self) -> None:
        assert not _is_refusal("Sure, here is the information you requested.")

    def test_empty_string(self) -> None:
        assert not _is_refusal("")


class TestIsViolationFinishReason:
    def test_violation_detected(self) -> None:
        data: dict[str, object] = {
            "choices": [{"finish_reason": "violation", "message": {"content": ""}}],
        }
        assert _is_violation_finish_reason(data)

    def test_stop_not_violation(self) -> None:
        data: dict[str, object] = {
            "choices": [{"finish_reason": "stop", "message": {"content": "ok"}}],
        }
        assert not _is_violation_finish_reason(data)

    def test_no_choices(self) -> None:
        assert not _is_violation_finish_reason({})

    def test_empty_choices(self) -> None:
        data: dict[str, object] = {"choices": []}
        assert not _is_violation_finish_reason(data)

    def test_invalid_choices_type(self) -> None:
        data: dict[str, object] = {"choices": "not a list"}
        assert not _is_violation_finish_reason(data)

    def test_non_mapping_choice_is_ignored(self) -> None:
        data: dict[str, object] = {"choices": ["bad"]}
        assert not _is_violation_finish_reason(data)


class TestBuildAuthHeaders:
    def test_bearer_token_default(self, endpoint: ApiEvalEndpoint) -> None:
        headers = _build_auth_headers(endpoint, "sk-test")
        assert headers["Authorization"] == "Bearer sk-test"
        assert "grayswan-api-key" not in headers

    def test_custom_auth_header(
        self, grayswan_endpoint: ApiEvalEndpoint,
    ) -> None:
        headers = _build_auth_headers(grayswan_endpoint, "gs-key-123")
        assert headers["grayswan-api-key"] == "gs-key-123"
        assert "Authorization" not in headers

    def test_content_type_always_present(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        headers = _build_auth_headers(endpoint, "sk-test")
        assert headers["Content-Type"] == "application/json"


class TestBuildMessages:
    def test_basic_messages(self) -> None:
        msgs = _build_messages("Hello", "System prompt", None)
        assert len(msgs) == 2
        assert msgs[0] == {"role": "system", "content": "System prompt"}
        assert msgs[1] == {"role": "user", "content": "Hello"}

    def test_no_system_prompt(self) -> None:
        msgs = _build_messages("Hello", None, None)
        assert len(msgs) == 1
        assert msgs[0] == {"role": "user", "content": "Hello"}

    def test_with_history(self) -> None:
        history = [
            {"role": "user", "content": "Previous question"},
            {"role": "assistant", "content": "Previous answer"},
        ]
        msgs = _build_messages("New question", "System", history)
        assert len(msgs) == 4
        assert msgs[0]["role"] == "system"
        assert msgs[1]["role"] == "user"
        assert msgs[1]["content"] == "Previous question"
        assert msgs[2]["role"] == "assistant"
        assert msgs[3]["role"] == "user"
        assert msgs[3]["content"] == "New question"


class TestExtractContent:
    def test_normal_response(self) -> None:
        data: dict[str, object] = {
            "choices": [{"message": {"content": "Hello!"}}],
        }
        assert _extract_content(data) == "Hello!"

    def test_empty_choices(self) -> None:
        data: dict[str, object] = {"choices": []}
        assert _extract_content(data) == ""

    def test_no_choices_key(self) -> None:
        assert _extract_content({}) == ""

    def test_missing_message(self) -> None:
        data: dict[str, object] = {"choices": [{}]}
        assert _extract_content(data) == ""

    def test_non_mapping_choice_returns_empty(self) -> None:
        data: dict[str, object] = {"choices": ["bad"]}
        assert _extract_content(data) == ""

    def test_non_mapping_message_returns_empty(self) -> None:
        data: dict[str, object] = {"choices": [{"message": "bad"}]}
        assert _extract_content(data) == ""

    def test_reasoning_fallback(self) -> None:
        data: dict[str, object] = {
            "choices": [{"message": {"reasoning": "thoughtful output"}}],
        }
        assert _extract_content(data) == "thoughtful output"


class TestCallChatApi:
    def test_successful_call(self, endpoint: ApiEvalEndpoint) -> None:
        mock_response = _mock_api_response("Sure, here you go.")

        with patch("urllib.request.urlopen", return_value=mock_response):
            text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert text == "Sure, here you go."
        assert not is_refused

    def test_refusal_response(self, endpoint: ApiEvalEndpoint) -> None:
        mock_response = _mock_api_response("I cannot help with that.")

        with patch("urllib.request.urlopen", return_value=mock_response):
            _text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert is_refused

    def test_violation_finish_reason(self, endpoint: ApiEvalEndpoint) -> None:
        mock_response = _mock_api_response("", finish_reason="violation")

        with patch("urllib.request.urlopen", return_value=mock_response):
            _text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert is_refused

    def test_empty_choices_returns_refusal(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        response_body = json.dumps({"choices": []}).encode()

        mock_response = MagicMock()
        mock_response.read.return_value = response_body
        mock_response.__enter__ = lambda s: s
        mock_response.__exit__ = MagicMock(return_value=False)

        with patch("urllib.request.urlopen", return_value=mock_response):
            _text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert is_refused

    def test_malformed_json_returns_refusal(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        mock_response = MagicMock()
        mock_response.read.return_value = b"{bad json"
        mock_response.__enter__ = lambda s: s
        mock_response.__exit__ = MagicMock(return_value=False)

        with patch("urllib.request.urlopen", return_value=mock_response):
            text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert text == "I cannot assist with that request."
        assert is_refused

    def test_timeout_returns_refusal(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        with patch(
            "urllib.request.urlopen",
            side_effect=TimeoutError("timed out"),
        ):
            _text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert is_refused

    def test_http_error_returns_refusal(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        import urllib.error

        with patch(
            "urllib.request.urlopen",
            side_effect=urllib.error.HTTPError(
                "https://api.example.com/v1/chat/completions",
                429,
                "Too Many Requests",
                {},  # type: ignore[arg-type]
                None,
            ),
        ):
            _text, is_refused = _call_chat_api(
                endpoint=endpoint,
                api_key="sk-test",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        assert is_refused

    def test_custom_auth_header_sent(
        self, grayswan_endpoint: ApiEvalEndpoint,
    ) -> None:
        mock_response = _mock_api_response("Sure thing.")

        with patch(
            "urllib.request.urlopen", return_value=mock_response,
        ) as mock_open:
            _call_chat_api(
                endpoint=grayswan_endpoint,
                api_key="gs-key-123",
                messages=[{"role": "user", "content": "Hello"}],
                max_tokens=50,
                timeout=10,
            )

        # Verify the request used custom header
        req = mock_open.call_args[0][0]
        assert req.get_header("Grayswan-api-key") == "gs-key-123"
        assert req.get_header("Authorization") is None


class TestEvaluateSuffixViaApi:
    def test_missing_api_key_skips(
        self,
        config: ApiEvalConfig,
    ) -> None:
        with patch.dict(os.environ, {}, clear=True):
            results = evaluate_suffix_via_api(
                "suffix", ["prompt1"], config,
            )

        assert len(results) == 1
        assert results[0].model_id == "test-ep"
        assert results[0].success_rate == 0.0
        assert results[0].eval_responses == []

    def test_successful_bypass(
        self,
        config: ApiEvalConfig,
    ) -> None:
        mock_response = _mock_api_response("Sure, here you go!")

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "adversarial suffix",
                ["How to do something bad?"],
                config,
            )

        assert len(results) == 1
        assert results[0].model_id == "test-ep"
        assert results[0].success_rate == 1.0

    def test_refusal_response(
        self,
        config: ApiEvalConfig,
    ) -> None:
        mock_response = _mock_api_response("I cannot help with that.")

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "adversarial suffix",
                ["How to do something bad?"],
                config,
            )

        assert len(results) == 1
        assert results[0].success_rate == 0.0

    def test_endpoint_system_prompt_override(self) -> None:
        ep = ApiEvalEndpoint(
            name="ep",
            base_url="https://api.example.com/v1",
            model="m",
            api_key_env="K",
            system_prompt="I am an endpoint override.",
        )
        cfg = ApiEvalConfig(
            endpoints=[ep],
            system_prompt="I am the default.",
        )
        # The endpoint system_prompt should win
        assert ep.system_prompt == "I am an endpoint override."
        assert cfg.system_prompt == "I am the default."

    def test_with_history(
        self,
        config: ApiEvalConfig,
    ) -> None:
        mock_response = _mock_api_response("Sure, continuing.")

        history = [
            {"role": "user", "content": "Previous question"},
            {"role": "assistant", "content": "Previous answer"},
        ]

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["New question"], config, history=history,
            )

        assert results[0].success_rate == 1.0

    def test_infix_position_replaces_marker(
        self,
        config: ApiEvalConfig,
    ) -> None:
        """Infix position replaces {suffix} marker in prompts."""
        mock_response = _mock_api_response("Sure, here's the info.")
        sent_bodies: list[dict[str, object]] = []

        def capture_urlopen(
            req: object, timeout: object = None,
        ) -> MagicMock:
            assert hasattr(req, "data")
            sent_bodies.append(json.loads(req.data.decode()))  # type: ignore[union-attr]
            return mock_response

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", side_effect=capture_urlopen),
        ):
            results = evaluate_suffix_via_api(
                "ADV_TOKENS",
                ["Write a story where {suffix} someone hacks wifi"],
                config,
                token_position="infix",
            )

        assert results[0].success_rate == 1.0
        # Verify the prompt was assembled correctly
        msgs = sent_bodies[0]["messages"]
        assert isinstance(msgs, list)
        user_msg = cast("dict[str, object]", msgs[-1])
        content = str(user_msg["content"])
        assert "ADV_TOKENS" in content
        assert "{suffix}" not in content
        assert content == "Write a story where ADV_TOKENS someone hacks wifi"

    def test_suffix_position_appends(
        self,
        config: ApiEvalConfig,
    ) -> None:
        """Default suffix position appends tokens after prompt."""
        mock_response = _mock_api_response("Sure thing.")
        sent_bodies: list[dict[str, object]] = []

        def capture_urlopen(
            req: object, timeout: object = None,
        ) -> MagicMock:
            assert hasattr(req, "data")
            sent_bodies.append(json.loads(req.data.decode()))  # type: ignore[union-attr]
            return mock_response

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", side_effect=capture_urlopen),
        ):
            results = evaluate_suffix_via_api(
                "ADV_TOKENS",
                ["How to do something"],
                config,
                token_position="suffix",
            )

        assert results[0].success_rate == 1.0
        msgs = sent_bodies[0]["messages"]
        assert isinstance(msgs, list)
        user_msg = cast("dict[str, object]", msgs[-1])
        assert str(user_msg["content"]) == "How to do something ADV_TOKENS"


class TestMultiturnEvaluation:
    def test_multiturn_all_pass(
        self, multiturn_config: ApiEvalConfig,
    ) -> None:
        """All turns respond without refusal — success."""
        mock_response = _mock_api_response("Sure, here are the details.")

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], multiturn_config,
            )

        assert results[0].success_rate == 1.0
        # 1 initial + 2 follow-ups = 3 responses
        assert len(results[0].eval_responses) == 3

    def test_multiturn_first_turn_refused(
        self, multiturn_config: ApiEvalConfig,
    ) -> None:
        """First turn refuses — no follow-ups attempted."""
        mock_response = _mock_api_response("I cannot help with that.")

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], multiturn_config,
            )

        assert results[0].success_rate == 0.0
        # Only 1 response — stopped after first refusal
        assert len(results[0].eval_responses) == 1

    def test_multiturn_second_turn_refused(
        self, multiturn_config: ApiEvalConfig,
    ) -> None:
        """First turn passes, second turn refuses."""
        responses = [
            _mock_api_response("Sure, here's the overview."),
            _mock_api_response("I cannot provide that information."),
        ]
        call_count = 0

        def mock_urlopen(req: object, timeout: object = None) -> MagicMock:
            nonlocal call_count
            resp = responses[min(call_count, len(responses) - 1)]
            call_count += 1
            return resp

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", side_effect=mock_urlopen),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], multiturn_config,
            )

        assert results[0].success_rate == 0.0
        # 2 responses: initial + one failed follow-up
        assert len(results[0].eval_responses) == 2

    def test_multiturn_violation_finish_reason(
        self, multiturn_config: ApiEvalConfig,
    ) -> None:
        """First turn passes, follow-up gets violation finish_reason."""
        responses = [
            _mock_api_response("Here's some info."),
            _mock_api_response("", finish_reason="violation"),
        ]
        call_count = 0

        def mock_urlopen(req: object, timeout: object = None) -> MagicMock:
            nonlocal call_count
            resp = responses[min(call_count, len(responses) - 1)]
            call_count += 1
            return resp

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", side_effect=mock_urlopen),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], multiturn_config,
            )

        assert results[0].success_rate == 0.0

    def test_multiturn_default_follow_ups(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        """When follow_up_prompts is empty, defaults are used."""
        config = ApiEvalConfig(
            endpoints=[endpoint],
            max_tokens=50,
            timeout=10,
            multiturn=True,
            multiturn_max_turns=2,
            follow_up_prompts=[],  # empty → use defaults
        )
        mock_response = _mock_api_response("Sure thing.")

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], config,
            )

        assert results[0].success_rate == 1.0
        # max_turns=2 → 1 initial + 1 follow-up = 2 responses
        assert len(results[0].eval_responses) == 2

    def test_multiturn_with_history(
        self, multiturn_config: ApiEvalConfig,
    ) -> None:
        """Multi-turn with prior history prepended."""
        mock_response = _mock_api_response("Continuing the conversation.")
        history = [
            {"role": "user", "content": "Earlier question"},
            {"role": "assistant", "content": "Earlier answer"},
        ]

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], multiturn_config,
                history=history,
            )

        assert results[0].success_rate == 1.0

    def test_multiturn_max_turns_caps_follow_ups(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        """multiturn_max_turns=1 means only initial turn, no follow-ups."""
        config = ApiEvalConfig(
            endpoints=[endpoint],
            max_tokens=50,
            timeout=10,
            multiturn=True,
            multiturn_max_turns=1,
            follow_up_prompts=["Follow up 1", "Follow up 2"],
        )
        mock_response = _mock_api_response("Sure thing.")

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            results = evaluate_suffix_via_api(
                "suffix", ["Test prompt"], config,
            )

        # max_turns=1 → no follow-ups, just the initial turn
        assert len(results[0].eval_responses) == 1
        assert results[0].success_rate == 1.0

    def test_multiturn_messages_accumulate(
        self, endpoint: ApiEvalEndpoint,
    ) -> None:
        """Verify the request bodies carry the correct accumulated messages."""
        config = ApiEvalConfig(
            endpoints=[endpoint],
            max_tokens=50,
            timeout=10,
            system_prompt="Be helpful.",
            multiturn=True,
            multiturn_max_turns=2,
            follow_up_prompts=["Give details."],
        )
        history = [
            {"role": "user", "content": "Prior Q"},
            {"role": "assistant", "content": "Prior A"},
        ]

        mock_response = _mock_api_response("Sure thing.")
        sent_bodies: list[dict[str, object]] = []

        def capture_urlopen(
            req: object, timeout: object = None,
        ) -> MagicMock:
            assert hasattr(req, "data")
            sent_bodies.append(json.loads(req.data.decode()))  # type: ignore[union-attr]
            return mock_response

        with (
            patch.dict(os.environ, {"TEST_API_KEY": "sk-test"}),
            patch("urllib.request.urlopen", side_effect=capture_urlopen),
        ):
            evaluate_suffix_via_api(
                "suffix", ["Attack prompt"], config, history=history,
            )

        assert len(sent_bodies) == 2

        # Turn 1: system + history + user prompt
        msgs_t1_raw = sent_bodies[0]["messages"]
        assert isinstance(msgs_t1_raw, list)
        msgs_t1: list[dict[str, str]] = [
            {str(k): str(v) for k, v in m.items()}
            for m in msgs_t1_raw
            if isinstance(m, dict)
        ]
        assert len(msgs_t1) == 4
        assert msgs_t1[0] == {"role": "system", "content": "Be helpful."}
        assert msgs_t1[1] == {"role": "user", "content": "Prior Q"}
        assert msgs_t1[2] == {"role": "assistant", "content": "Prior A"}
        assert msgs_t1[3]["role"] == "user"
        assert "Attack prompt" in msgs_t1[3]["content"]
        assert "suffix" in msgs_t1[3]["content"]

        # Turn 2: same as turn 1 + assistant response + follow-up
        msgs_t2 = sent_bodies[1]["messages"]
        assert isinstance(msgs_t2, list)
        assert len(msgs_t2) == 6
        # First 4 should match turn 1
        assert msgs_t2[:4] == msgs_t1
        # Then assistant response from turn 1
        assert msgs_t2[4] == {"role": "assistant", "content": "Sure thing."}
        # Then the follow-up prompt
        assert msgs_t2[5] == {"role": "user", "content": "Give details."}


# ── Parser tests for standalone api_eval fields ──────────────────────


def _minimal_api_eval_raw(
    **overrides: object,
) -> dict[str, object]:
    """Build a minimal valid [api_eval] TOML dict."""
    base: dict[str, object] = {
        "endpoints": [{
            "name": "test",
            "base_url": "https://api.example.com/v1",
            "model": "test-model",
            "api_key_env": "TEST_KEY",
        }],
    }
    base.update(overrides)
    return {"api_eval": base}


class TestParseApiEvalStandaloneFields:
    """Tests for parsing the new standalone api_eval fields."""

    def test_token_text_parsed(self) -> None:
        raw = _minimal_api_eval_raw(
            token_text="adversarial tokens",
            prompts=["Test prompt {suffix} here"],
        )
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.token_text == "adversarial tokens"

    def test_token_position_default(self) -> None:
        raw = _minimal_api_eval_raw()
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.token_position == "suffix"

    def test_token_position_infix(self) -> None:
        raw = _minimal_api_eval_raw(token_position="infix")
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.token_position == "infix"

    def test_token_position_invalid(self) -> None:
        raw = _minimal_api_eval_raw(token_position="middle")
        with pytest.raises(ValueError, match="token_position"):
            _parse_api_eval(raw)

    def test_prompts_parsed(self) -> None:
        raw = _minimal_api_eval_raw(
            prompts=["Prompt A", "Prompt B"],
        )
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.prompts == ["Prompt A", "Prompt B"]

    def test_prompts_default_empty(self) -> None:
        raw = _minimal_api_eval_raw()
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.prompts == []

    def test_token_text_requires_prompts(self) -> None:
        raw = _minimal_api_eval_raw(token_text="tokens")
        with pytest.raises(ValueError, match="prompts must be non-empty"):
            _parse_api_eval(raw)

    def test_token_text_empty_rejected(self) -> None:
        raw = _minimal_api_eval_raw(token_text="")
        with pytest.raises(ValueError, match="non-empty"):
            _parse_api_eval(raw)

    def test_token_text_not_string_rejected(self) -> None:
        raw = _minimal_api_eval_raw(token_text=42)
        with pytest.raises(TypeError, match="token_text"):
            _parse_api_eval(raw)

    def test_prompts_not_list_rejected(self) -> None:
        raw = _minimal_api_eval_raw(prompts="not a list")
        with pytest.raises(TypeError, match="prompts must be a list"):
            _parse_api_eval(raw)

    def test_prompts_item_not_string_rejected(self) -> None:
        raw = _minimal_api_eval_raw(prompts=["ok", 42])
        with pytest.raises(TypeError, match=r"prompts\[1\]"):
            _parse_api_eval(raw)


class TestParseDefenseProxyFields:
    """Tests for parsing the defense_proxy fields."""

    def test_defense_proxy_sic(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy="sic")
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy == "sic"

    def test_defense_proxy_cast(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy="cast")
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy == "cast"

    def test_defense_proxy_both(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy="both")
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy == "both"

    def test_defense_proxy_absent(self) -> None:
        raw = _minimal_api_eval_raw()
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy is None

    def test_defense_proxy_invalid(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy="firewall")
        with pytest.raises(ValueError, match="defense_proxy"):
            _parse_api_eval(raw)

    def test_defense_proxy_not_string(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy=42)
        with pytest.raises(TypeError, match="defense_proxy"):
            _parse_api_eval(raw)

    def test_defense_proxy_sic_mode_default(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy="sic")
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy_sic_mode == "direction"

    def test_defense_proxy_sic_mode_generation(self) -> None:
        raw = _minimal_api_eval_raw(
            defense_proxy="sic",
            defense_proxy_sic_mode="generation",
        )
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy_sic_mode == "generation"

    def test_defense_proxy_sic_mode_invalid(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_sic_mode="foobar")
        with pytest.raises(ValueError, match="defense_proxy_sic_mode"):
            _parse_api_eval(raw)

    def test_defense_proxy_cast_mode_default(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy="cast")
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy_cast_mode == "gate"

    def test_defense_proxy_cast_mode_full(self) -> None:
        raw = _minimal_api_eval_raw(
            defense_proxy="cast",
            defense_proxy_cast_mode="full",
        )
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy_cast_mode == "full"

    def test_defense_proxy_cast_mode_invalid(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_cast_mode="half")
        with pytest.raises(ValueError, match="defense_proxy_cast_mode"):
            _parse_api_eval(raw)

    def test_defense_proxy_cast_layers(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_cast_layers=[14, 15])
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy_cast_layers == [14, 15]

    def test_defense_proxy_cast_layers_not_list(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_cast_layers="14")
        with pytest.raises(TypeError, match="defense_proxy_cast_layers"):
            _parse_api_eval(raw)

    def test_defense_proxy_cast_layers_item_not_int(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_cast_layers=[14, "x"])
        with pytest.raises(TypeError, match=r"defense_proxy_cast_layers\[1\]"):
            _parse_api_eval(raw)

    def test_defense_proxy_sic_max_iterations_bounds(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_sic_max_iterations=0)
        with pytest.raises(ValueError, match="defense_proxy_sic_max_iterations"):
            _parse_api_eval(raw)

    def test_defense_proxy_cast_max_tokens_bounds(self) -> None:
        raw = _minimal_api_eval_raw(defense_proxy_cast_max_tokens=0)
        with pytest.raises(ValueError, match="defense_proxy_cast_max_tokens"):
            _parse_api_eval(raw)

    def test_defense_proxy_numeric_fields(self) -> None:
        raw = _minimal_api_eval_raw(
            defense_proxy="both",
            defense_proxy_sic_threshold=0.5,
            defense_proxy_cast_threshold=0.3,
            defense_proxy_cast_alpha=2.0,
        )
        cfg = _parse_api_eval(raw)
        assert cfg is not None
        assert cfg.defense_proxy_sic_threshold == 0.5
        assert cfg.defense_proxy_cast_threshold == 0.3
        assert cfg.defense_proxy_cast_alpha == 2.0


class TestStandaloneApiEvalPredicate:
    """Tests for the _has_standalone_api_eval predicate."""

    def test_returns_true_when_token_text_set(self) -> None:
        from vauban.config._mode_registry import _has_standalone_api_eval
        from vauban.types import PipelineConfig

        api_cfg = ApiEvalConfig(
            endpoints=[ApiEvalEndpoint(
                name="ep", base_url="https://api.example.com/v1",
                model="m", api_key_env="K",
            )],
            token_text="tokens",
            prompts=["prompt"],
        )
        config = PipelineConfig(
            model_path="",
            harmful_path="",  # type: ignore[arg-type]
            harmless_path="",  # type: ignore[arg-type]
            api_eval=api_cfg,
        )
        assert _has_standalone_api_eval(config) is True

    def test_returns_false_when_no_api_eval(self) -> None:
        from vauban.config._mode_registry import _has_standalone_api_eval
        from vauban.types import PipelineConfig

        config = PipelineConfig(
            model_path="some-model",
            harmful_path="",  # type: ignore[arg-type]
            harmless_path="",  # type: ignore[arg-type]
        )
        assert _has_standalone_api_eval(config) is False

    def test_returns_false_when_token_text_none(self) -> None:
        from vauban.config._mode_registry import _has_standalone_api_eval
        from vauban.types import PipelineConfig

        api_cfg = ApiEvalConfig(
            endpoints=[ApiEvalEndpoint(
                name="ep", base_url="https://api.example.com/v1",
                model="m", api_key_env="K",
            )],
        )
        config = PipelineConfig(
            model_path="some-model",
            harmful_path="",  # type: ignore[arg-type]
            harmless_path="",  # type: ignore[arg-type]
            api_eval=api_cfg,
        )
        assert _has_standalone_api_eval(config) is False


class TestModeApiEvalRunner:
    """Tests for the standalone api_eval mode runner."""

    def test_runner_calls_api_and_writes_report(
        self, tmp_path: Path,
    ) -> None:
        """Mode runner evaluates tokens and writes JSON report."""
        import time

        from vauban._pipeline._context import EarlyModeContext
        from vauban._pipeline._mode_api_eval import _run_api_eval_mode
        from vauban.types import PipelineConfig

        output_dir = tmp_path / "output"
        api_cfg = ApiEvalConfig(
            endpoints=[ApiEvalEndpoint(
                name="mock-ep",
                base_url="https://api.example.com/v1",
                model="test-model",
                api_key_env="TEST_KEY",
            )],
            token_text="adversarial tokens",
            token_position="suffix",
            prompts=["Test prompt"],
        )
        config = PipelineConfig(
            model_path="",
            harmful_path="",  # type: ignore[arg-type]
            harmless_path="",  # type: ignore[arg-type]
            api_eval=api_cfg,
            output_dir=output_dir,
        )
        ctx = EarlyModeContext(
            config_path="test.toml",
            config=config,
            model=None,
            tokenizer=None,
            t0=time.monotonic(),
        )

        mock_response = _mock_api_response("Sure, here you go!")

        with (
            patch.dict(os.environ, {"TEST_KEY": "sk-test"}),
            patch("urllib.request.urlopen", return_value=mock_response),
        ):
            _run_api_eval_mode(ctx)

        report_path = output_dir / "api_eval_report.json"
        assert report_path.exists()
        report = json.loads(report_path.read_text())
        assert report["token_text"] == "adversarial tokens"
        assert "mock-ep" in report["endpoints"]
        ep_data = report["endpoints"]["mock-ep"]
        assert ep_data["success_rate"] == 1.0
