"""
Tests for the LLMHarness convenience wrapper.
"""

from __future__ import annotations

from typing import Any, Dict

import pytest

from metamorphic_guard.llm_harness import LLMHarness


def test_llm_harness_passes_role_specific_configs(monkeypatch):
    """LLMHarness should forward distinct baseline/candidate configs to run_eval."""

    captured: Dict[str, Any] = {}

    def fake_run_eval(*args, **kwargs):
        captured["kwargs"] = kwargs
        return {
            "baseline": {
                "passes": 1,
                "total": 1,
                "pass_rate": 1.0,
                "prop_violations": [],
                "mr_violations": [],
            },
            "candidate": {
                "passes": 1,
                "total": 1,
                "pass_rate": 1.0,
                "prop_violations": [],
                "mr_violations": [],
            },
            "delta_ci": [0.0, 0.0],
            "decision": {"adopt": True, "reason": "meets_gate"},
            "statistics": {},
        }

    monkeypatch.setattr("metamorphic_guard.api.run_eval", fake_run_eval)

    harness = LLMHarness(model="gpt-4", provider="openai", executor_config={"api_key": "test-key"})
    result = harness.run(
        case={"system": "candidate-system", "user": "hello"},
        baseline_model="gpt-3.5-turbo",
        baseline_system="baseline-system",
        n=1,
        seed=123,
        bootstrap=False,
    )

    kwargs = captured["kwargs"]
    assert kwargs["baseline_executor_config"]["model"] == "gpt-3.5-turbo"
    assert kwargs["baseline_executor_config"]["system_prompt"] == "baseline-system"
    assert kwargs["candidate_executor_config"]["model"] == "gpt-4"
    assert kwargs["candidate_executor_config"]["system_prompt"] == "candidate-system"
    assert kwargs["baseline_executor"] == "openai"
    assert kwargs["candidate_executor"] == "openai"
    assert kwargs["executor"] == "openai"

    llm_metrics = result.get("llm_metrics")
    assert llm_metrics is not None
    assert "baseline" in llm_metrics and "candidate" in llm_metrics


def test_llm_harness_retains_llm_metrics(monkeypatch):
    """LLMHarness should keep harness-provided llm_metrics intact."""

    metrics_payload = {
        "baseline": {"count": 1, "total_cost_usd": 0.05, "total_tokens": 120, "retry_total": 0},
        "candidate": {"count": 1, "total_cost_usd": 0.04, "total_tokens": 110, "retry_total": 0},
        "cost_delta_usd": -0.01,
        "cost_ratio": 0.8,
    }

    def fake_run_eval(*args, **kwargs):
        return {
            "baseline": {"passes": 1, "total": 1, "pass_rate": 1.0, "prop_violations": [], "mr_violations": []},
            "candidate": {"passes": 1, "total": 1, "pass_rate": 1.0, "prop_violations": [], "mr_violations": []},
            "delta_ci": [0.0, 0.0],
            "decision": {"adopt": True, "reason": "meets_gate"},
            "statistics": {},
            "llm_metrics": metrics_payload,
        }

    monkeypatch.setattr("metamorphic_guard.api.run_eval", fake_run_eval)

    harness = LLMHarness(model="gpt-4", provider="openai", executor_config={"api_key": "test-key"})
    result = harness.run(case="hello", bootstrap=False)

    assert result["llm_metrics"]["cost_delta_usd"] == pytest.approx(-0.01, rel=1e-6)
    assert result["llm_metrics"]["cost_ratio"] == pytest.approx(0.8, rel=1e-6)


class TestLLMHarnessIntegration:
    """End-to-end integration tests for LLM evaluation."""

    def test_full_evaluation_flow(self, monkeypatch):
        """Test complete evaluation flow with mocked executors."""
        from metamorphic_guard.judges.builtin import LengthJudge
        from metamorphic_guard.mutants.builtin import ParaphraseMutant
        
        # Mock executor responses
        def mock_execute(self, file_path, func_name, args, timeout_s, mem_mb, **kwargs):
            prompt = args[0] if args else ""
            return {
                "success": True,
                "result": f"Response to: {prompt}",
                "tokens_prompt": 10,
                "tokens_completion": 20,
                "tokens_total": 30,
                "cost_usd": 0.0001,
                "duration_ms": 500.0,
                "finish_reason": "stop",
                "retries": 0,
            }
        
        # Mock openai module to prevent ImportError in __init__
        from unittest.mock import MagicMock
        mock_openai = MagicMock()
        mock_openai.OpenAI = MagicMock()
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.openai",
            mock_openai,
        )
        
        # Mock OpenAI executor __init__ to bypass API key check
        original_init = None
        try:
            from metamorphic_guard.executors.openai import OpenAIExecutor
            original_init = OpenAIExecutor.__init__
            
            def mock_init(self, config=None):
                # Call parent __init__ but skip API key validation
                from metamorphic_guard.executors import Executor
                Executor.__init__(self, config)
                cfg = config or {}
                self.provider = cfg.get("provider", "openai")
                self.model = cfg.get("model", "gpt-3.5-turbo")
                self.max_tokens = cfg.get("max_tokens", 512)
                self.temperature = cfg.get("temperature", 0.0)
                self.seed = cfg.get("seed")
                self.system_prompt = cfg.get("system_prompt")
                self.max_retries = int(cfg.get("max_retries", 3))
                # Set api_key to a dummy value to bypass validation
                if config and "api_key" in config:
                    self.api_key = config["api_key"]
                else:
                    self.api_key = "test-key"
                self.client = mock_openai.OpenAI(api_key=self.api_key)
            
            monkeypatch.setattr(
                "metamorphic_guard.executors.openai.OpenAIExecutor.__init__",
                mock_init,
            )
        except ImportError:
            pass
        
        # Mock OpenAI executor execute method
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.OpenAIExecutor.execute",
            mock_execute,
        )
        
        harness = LLMHarness(
            model="gpt-3.5-turbo",
            provider="openai",
            executor_config={"api_key": "test-key"},
        )
        
        result = harness.run(
            case="Test prompt",
            props=[LengthJudge(config={"min_chars": 5})],
            mrs=[ParaphraseMutant()],
            n=5,
            seed=42,
            bootstrap=False,
        )
        
        assert "baseline" in result
        assert "candidate" in result
        assert "decision" in result
        assert "llm_metrics" in result

    def test_error_handling_and_retries(self, monkeypatch):
        """Test error handling and retry logic in integration."""
        attempt_count = {"count": 0}
        
        def mock_execute_with_retry(self, file_path, func_name, args, timeout_s, mem_mb, **kwargs):
            attempt_count["count"] += 1
            if attempt_count["count"] < 2:
                return {
                    "success": False,
                    "error": "Rate limit exceeded",
                    "error_code": "rate_limit",
                    "retries": attempt_count["count"] - 1,
                }
            return {
                "success": True,
                "result": "Success after retry",
                "tokens_prompt": 5,
                "tokens_completion": 10,
                "tokens_total": 15,
                "cost_usd": 0.00005,
                "duration_ms": 200.0,
                "finish_reason": "stop",
                "retries": 1,
            }
        
        # Mock openai module to prevent ImportError in __init__
        from unittest.mock import MagicMock
        mock_openai = MagicMock()
        mock_openai.OpenAI = MagicMock()
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.openai",
            mock_openai,
        )
        
        # Mock OpenAI executor __init__ to bypass API key check
        try:
            from metamorphic_guard.executors.openai import OpenAIExecutor
            
            def mock_init(self, config=None):
                # Call parent __init__ but skip API key validation
                from metamorphic_guard.executors import Executor
                Executor.__init__(self, config)
                cfg = config or {}
                self.provider = cfg.get("provider", "openai")
                self.model = cfg.get("model", "gpt-3.5-turbo")
                self.max_tokens = cfg.get("max_tokens", 512)
                self.temperature = cfg.get("temperature", 0.0)
                self.seed = cfg.get("seed")
                self.system_prompt = cfg.get("system_prompt")
                self.max_retries = int(cfg.get("max_retries", 3))
                # Set api_key to a dummy value to bypass validation
                if config and "api_key" in config:
                    self.api_key = config["api_key"]
                else:
                    self.api_key = "test-key"
                self.client = mock_openai.OpenAI(api_key=self.api_key)
            
            monkeypatch.setattr(
                "metamorphic_guard.executors.openai.OpenAIExecutor.__init__",
                mock_init,
            )
        except ImportError:
            pass
        
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.OpenAIExecutor.execute",
            mock_execute_with_retry,
        )
        
        harness = LLMHarness(
            model="gpt-3.5-turbo",
            provider="openai",
            executor_config={"api_key": "test-key", "max_retries": 3},
        )
        
        result = harness.run(case="Test", n=1, seed=42, bootstrap=False)
        
        # Should eventually succeed after retry
        assert result["candidate"]["total"] == 1
        llm_metrics = result.get("llm_metrics", {})
        candidate_metrics = llm_metrics.get("candidate", {})
        assert candidate_metrics.get("retry_total", 0) >= 1

    def test_cost_tracking(self, monkeypatch):
        """Test that cost is properly tracked across evaluations."""
        def mock_execute(self, file_path, func_name, args, timeout_s, mem_mb, **kwargs):
            return {
                "success": True,
                "result": "Response",
                "tokens_prompt": 100,
                "tokens_completion": 200,
                "tokens_total": 300,
                "cost_usd": 0.001,
                "duration_ms": 1000.0,
                "finish_reason": "stop",
                "retries": 0,
            }
        
        # Mock openai module to prevent ImportError in __init__
        from unittest.mock import MagicMock
        mock_openai = MagicMock()
        mock_openai.OpenAI = MagicMock()
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.openai",
            mock_openai,
        )
        
        # Mock OpenAI executor __init__ to bypass API key check
        try:
            from metamorphic_guard.executors.openai import OpenAIExecutor
            
            def mock_init(self, config=None):
                # Call parent __init__ but skip API key validation
                from metamorphic_guard.executors import Executor
                Executor.__init__(self, config)
                cfg = config or {}
                self.provider = cfg.get("provider", "openai")
                self.model = cfg.get("model", "gpt-3.5-turbo")
                self.max_tokens = cfg.get("max_tokens", 512)
                self.temperature = cfg.get("temperature", 0.0)
                self.seed = cfg.get("seed")
                self.system_prompt = cfg.get("system_prompt")
                self.max_retries = int(cfg.get("max_retries", 3))
                # Set api_key to a dummy value to bypass validation
                if config and "api_key" in config:
                    self.api_key = config["api_key"]
                else:
                    self.api_key = "test-key"
                self.client = mock_openai.OpenAI(api_key=self.api_key)
            
            monkeypatch.setattr(
                "metamorphic_guard.executors.openai.OpenAIExecutor.__init__",
                mock_init,
            )
        except ImportError:
            pass
        
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.OpenAIExecutor.execute",
            mock_execute,
        )
        
        harness = LLMHarness(
            model="gpt-4",
            provider="openai",
            executor_config={"api_key": "test-key"},
        )
        
        result = harness.run(case="Test", n=10, seed=42, bootstrap=False)
        
        llm_metrics = result.get("llm_metrics", {})
        assert "baseline" in llm_metrics
        assert "candidate" in llm_metrics
        assert "cost_delta_usd" in llm_metrics
        
        # Should have tracked costs
        baseline_cost = llm_metrics["baseline"].get("total_cost_usd", 0)
        candidate_cost = llm_metrics["candidate"].get("total_cost_usd", 0)
        assert baseline_cost > 0
        assert candidate_cost > 0

    def test_multiple_model_comparison(self, monkeypatch):
        """Test comparison between different models."""
        def mock_execute(self, file_path, func_name, args, timeout_s, mem_mb, **kwargs):
            model = func_name or "gpt-3.5-turbo"
            # Simulate different costs for different models
            if "gpt-4" in model:
                cost = 0.002
            else:
                cost = 0.0005
            
            return {
                "success": True,
                "result": f"Response from {model}",
                "tokens_prompt": 50,
                "tokens_completion": 100,
                "tokens_total": 150,
                "cost_usd": cost,
                "duration_ms": 800.0,
                "finish_reason": "stop",
                "retries": 0,
            }
        
        # Mock openai module to prevent ImportError in __init__
        from unittest.mock import MagicMock
        mock_openai = MagicMock()
        mock_openai.OpenAI = MagicMock()
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.openai",
            mock_openai,
        )
        
        # Mock OpenAI executor __init__ to bypass API key check
        try:
            from metamorphic_guard.executors.openai import OpenAIExecutor
            
            def mock_init(self, config=None):
                # Call parent __init__ but skip API key validation
                from metamorphic_guard.executors import Executor
                Executor.__init__(self, config)
                cfg = config or {}
                self.provider = cfg.get("provider", "openai")
                self.model = cfg.get("model", "gpt-3.5-turbo")
                self.max_tokens = cfg.get("max_tokens", 512)
                self.temperature = cfg.get("temperature", 0.0)
                self.seed = cfg.get("seed")
                self.system_prompt = cfg.get("system_prompt")
                self.max_retries = int(cfg.get("max_retries", 3))
                # Set api_key to a dummy value to bypass validation
                if config and "api_key" in config:
                    self.api_key = config["api_key"]
                else:
                    self.api_key = "test-key"
                self.client = mock_openai.OpenAI(api_key=self.api_key)
            
            monkeypatch.setattr(
                "metamorphic_guard.executors.openai.OpenAIExecutor.__init__",
                mock_init,
            )
        except ImportError:
            pass
        
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.OpenAIExecutor.execute",
            mock_execute,
        )
        
        harness = LLMHarness(
            model="gpt-4",
            provider="openai",
            executor_config={"api_key": "test-key"},
            baseline_model="gpt-3.5-turbo",
        )
        
        result = harness.run(case="Test", n=5, seed=42, bootstrap=False)
        
        llm_metrics = result.get("llm_metrics", {})
        cost_delta = llm_metrics.get("cost_delta_usd", 0)
        
        # GPT-4 should be more expensive (or at least cost tracking should work)
        # Cost delta may be 0 if both models have same cost in test, but metrics should exist
        assert "cost_delta_usd" in llm_metrics
        assert "baseline" in result
        assert "candidate" in result

    def test_system_prompt_handling(self, monkeypatch):
        """Test that system prompts are properly handled."""
        captured_prompts = []
        
        def mock_execute(self, file_path, func_name, args, timeout_s, mem_mb, **kwargs):
            # Extract system prompt from args if present (for LLM executors, system prompt may be in args)
            system_prompt = ""
            if args and len(args) > 1 and isinstance(args[1], str):
                system_prompt = args[1]
            captured_prompts.append({
                "system": system_prompt,
                "user": args[0] if args else "",
            })
            return {
                "success": True,
                "result": "Response",
                "tokens_prompt": 10,
                "tokens_completion": 5,
                "tokens_total": 15,
                "cost_usd": 0.00001,
                "duration_ms": 100.0,
                "finish_reason": "stop",
                "retries": 0,
            }
        
        # Mock openai module to prevent ImportError in __init__
        from unittest.mock import MagicMock
        mock_openai = MagicMock()
        mock_openai.OpenAI = MagicMock()
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.openai",
            mock_openai,
        )
        
        # Mock OpenAI executor __init__ to bypass API key check
        try:
            from metamorphic_guard.executors.openai import OpenAIExecutor
            
            def mock_init(self, config=None):
                # Call parent __init__ but skip API key validation
                from metamorphic_guard.executors import Executor
                Executor.__init__(self, config)
                cfg = config or {}
                self.provider = cfg.get("provider", "openai")
                self.model = cfg.get("model", "gpt-3.5-turbo")
                self.max_tokens = cfg.get("max_tokens", 512)
                self.temperature = cfg.get("temperature", 0.0)
                self.seed = cfg.get("seed")
                self.system_prompt = cfg.get("system_prompt")
                self.max_retries = int(cfg.get("max_retries", 3))
                # Set api_key to a dummy value to bypass validation
                if config and "api_key" in config:
                    self.api_key = config["api_key"]
                else:
                    self.api_key = "test-key"
                self.client = mock_openai.OpenAI(api_key=self.api_key)
            
            monkeypatch.setattr(
                "metamorphic_guard.executors.openai.OpenAIExecutor.__init__",
                mock_init,
            )
        except ImportError:
            pass
        
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.OpenAIExecutor.execute",
            mock_execute,
        )
        
        harness = LLMHarness(
            model="gpt-3.5-turbo",
            provider="openai",
            executor_config={"api_key": "test-key"},
        )
        
        result = harness.run(
            case={"system": "You are helpful", "user": "Hello"},
            n=2,
            seed=42,
            bootstrap=False,
        )
        
        # Should have captured system prompts
        assert len(captured_prompts) >= 2
        assert any("You are helpful" in p.get("system", "") for p in captured_prompts)

    def test_judge_and_mutant_integration(self, monkeypatch):
        """Test integration with judges and mutants."""
        from metamorphic_guard.judges.builtin import LengthJudge
        from metamorphic_guard.mutants.builtin import ParaphraseMutant
        
        def mock_execute(self, file_path, func_name, args, timeout_s, mem_mb, **kwargs):
            return {
                "success": True,
                "result": "A valid response with sufficient length",
                "tokens_prompt": 10,
                "tokens_completion": 10,
                "tokens_total": 20,
                "cost_usd": 0.00001,
                "duration_ms": 100.0,
                "finish_reason": "stop",
                "retries": 0,
            }
        
        # Mock openai module to prevent ImportError in __init__
        from unittest.mock import MagicMock
        mock_openai = MagicMock()
        mock_openai.OpenAI = MagicMock()
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.openai",
            mock_openai,
        )
        
        # Mock OpenAI executor __init__ to bypass API key check
        try:
            from metamorphic_guard.executors.openai import OpenAIExecutor
            
            def mock_init(self, config=None):
                # Call parent __init__ but skip API key validation
                from metamorphic_guard.executors import Executor
                Executor.__init__(self, config)
                cfg = config or {}
                self.provider = cfg.get("provider", "openai")
                self.model = cfg.get("model", "gpt-3.5-turbo")
                self.max_tokens = cfg.get("max_tokens", 512)
                self.temperature = cfg.get("temperature", 0.0)
                self.seed = cfg.get("seed")
                self.system_prompt = cfg.get("system_prompt")
                self.max_retries = int(cfg.get("max_retries", 3))
                # Set api_key to a dummy value to bypass validation
                if config and "api_key" in config:
                    self.api_key = config["api_key"]
                else:
                    self.api_key = "test-key"
                self.client = mock_openai.OpenAI(api_key=self.api_key)
            
            monkeypatch.setattr(
                "metamorphic_guard.executors.openai.OpenAIExecutor.__init__",
                mock_init,
            )
        except ImportError:
            pass
        
        monkeypatch.setattr(
            "metamorphic_guard.executors.openai.OpenAIExecutor.execute",
            mock_execute,
        )
        
        harness = LLMHarness(
            model="gpt-3.5-turbo",
            provider="openai",
            executor_config={"api_key": "test-key"},
        )
        
        result = harness.run(
            case="Test prompt",
            props=[
                LengthJudge(config={"min_chars": 10}),
                LengthJudge(config={"min_chars": 0}),  # RegexJudge not available, using LengthJudge
            ],
            mrs=[ParaphraseMutant()],
            n=5,
            seed=42,
            bootstrap=False,
        )
        
        # Should have evaluated properties and relations
        assert "baseline" in result
        assert "candidate" in result
        baseline = result["baseline"]
        candidate = result["candidate"]
        
        # Should have property and MR results
        assert "prop_violations" in baseline
        assert "mr_violations" in baseline
        assert "prop_violations" in candidate
        assert "mr_violations" in candidate

