"""
E2E Integration Tests for LLM providers.

Uses real API calls to test:
1. OpenAI direct API
2. OpenRouter API
3. Ollama Cloud API
4. OpenCode CLI passthrough

Environment variables required (set in .env or directly):
- OPENAI_API_KEY
- OPENROUTER_API_KEY
- OLLAMA_CLOUD_API_KEY

Run with: pytest tests/e2e/test_real_llm.py -v --run-e2e
"""

from __future__ import annotations

import os
import time
from dataclasses import dataclass
from pathlib import Path

import pytest

# Load .env file
try:
    from dotenv import load_dotenv

    load_dotenv(Path(__file__).parent.parent.parent / ".env")
except ImportError:
    pass  # dotenv not installed, rely on environment

from lattice.core.types.config import CompilerConfig
from lattice.shell.llm import llm_complete


# Test configurations
PROVIDERS = {
    "openai": {
        "model": "openai/gpt-5-mini",
        "api_key_env": "OPENAI_API_KEY",
        "skip_if_missing": "OPENAI_API_KEY",
    },
    "openrouter": {
        "model": "openrouter/openai/gpt-5-nano",
        "api_key_env": "OPENROUTER_API_KEY",
        "skip_if_missing": "OPENROUTER_API_KEY",
    },
    "ollama_cloud": {
        "model": "ollama/gpt-oss:20b",
        "api_key_env": "OLLAMA_CLOUD_API_KEY",
        "skip_if_missing": "OLLAMA_CLOUD_API_KEY",
    },
    "opencode_cli": {
        "model": "cli:opencode:openai/gpt-5-mini",
        "api_key_env": "",  # CLI mode doesn't need API key
        "skip_if_missing": None,  # Always try, skip if CLI not found
    },
}

# Test prompts for different scenarios
TEST_PROMPTS = {
    "simple": "Say 'hello' in exactly one word. No other text.",
    "json_format": 'Return a JSON object with exactly one key "greeting" and value "hello". Only the JSON, no other text.',
    "reasoning": "What is 2 + 3? Answer with just the number.",
    "creative": "Write a haiku about code. Exactly 3 lines.",
    "structured": "List 3 programming languages. Format: one per line, numbered 1-3.",
    "compiler_task": """Analyze the following session log and extract a rule proposal.

Session: user prefers type hints on all function parameters.

Output in this exact format:
<triage>
Key signals: [list key observations]
</triage>
<cross_ref>
Pattern: [describe pattern]
Confidence: HIGH/MEDIUM/LOW
</cross_ref>
<synthesis>
## PROPOSAL: [action] - [title]
Action: ADD/MERGE/REMOVE/REWORD
Target: [filename].md
Content: |
  [rule content]
Evidence: [session id]
Rationale: [explanation]
</synthesis>""",
}


@dataclass
class LLMResponse:
    """Structured LLM response for comparison."""

    provider: str
    prompt_type: str
    success: bool
    content: str
    latency_ms: float
    error: str | None = None
    response_length: int = 0

    def __post_init__(self):
        if self.success:
            self.response_length = len(self.content)


def call_llm(provider: str, prompt_type: str) -> LLMResponse:
    """Make a real LLM call and return structured result."""
    from returns.result import Failure, Success

    config_data = PROVIDERS[provider]
    prompt = TEST_PROMPTS[prompt_type]

    # Build config with optional base_url
    config_kwargs = {
        "model": config_data["model"],
        "api_key_env": config_data["api_key_env"],
    }

    # Check for base_url_env (for Ollama Cloud and other remote hosts)
    base_url_env = config_data.get("base_url_env")
    if base_url_env:
        base_url = os.environ.get(base_url_env)
        if base_url:
            config_kwargs["base_url"] = base_url

    config = CompilerConfig(**config_kwargs)

    start = time.perf_counter()
    result = llm_complete(config, prompt)
    latency_ms = (time.perf_counter() - start) * 1000

    if isinstance(result, Failure):
        return LLMResponse(
            provider=provider,
            prompt_type=prompt_type,
            success=False,
            content="",
            latency_ms=latency_ms,
            error=result.failure(),
        )

    return LLMResponse(
        provider=provider,
        prompt_type=prompt_type,
        success=True,
        content=result.unwrap(),
        latency_ms=latency_ms,
    )


# ============================================================================
# Provider Connectivity Tests
# ============================================================================


@pytest.mark.e2e
class TestProviderConnectivity:
    """Test each provider can make a simple call."""

    @pytest.mark.skipif(
        not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set"
    )
    def test_openai_connectivity(self):
        """OpenAI direct API should respond."""
        response = call_llm("openai", "simple")
        assert response.success, f"OpenAI failed: {response.error}"
        assert len(response.content) > 0
        assert response.latency_ms > 0
        print(f"\n✓ OpenAI: {response.content!r} ({response.latency_ms:.0f}ms)")

    @pytest.mark.skipif(
        not os.environ.get("OPENROUTER_API_KEY"), reason="OPENROUTER_API_KEY not set"
    )
    def test_openrouter_connectivity(self):
        """OpenRouter API should respond."""
        response = call_llm("openrouter", "simple")
        assert response.success, f"OpenRouter failed: {response.error}"
        assert len(response.content) > 0
        assert response.latency_ms > 0
        print(f"\n✓ OpenRouter: {response.content!r} ({response.latency_ms:.0f}ms)")

    @pytest.mark.skipif(
        not os.environ.get("OLLAMA_CLOUD_API_KEY"),
        reason="OLLAMA_CLOUD_API_KEY not set",
    )
    def test_ollama_cloud_connectivity(self):
        """Ollama Cloud API should respond."""
        response = call_llm("ollama_cloud", "simple")
        assert response.success, f"Ollama Cloud failed: {response.error}"
        assert len(response.content) > 0
        assert response.latency_ms > 0
        print(f"\n✓ Ollama Cloud: {response.content!r} ({response.latency_ms:.0f}ms)")

    def test_opencode_cli_connectivity(self):
        """OpenCode CLI should respond."""
        response = call_llm("opencode_cli", "simple")
        assert response.success, f"OpenCode CLI failed: {response.error}"
        assert len(response.content) > 0
        assert response.latency_ms > 0
        print(f"\n✓ OpenCode CLI: {response.content!r} ({response.latency_ms:.0f}ms)")


# ============================================================================
# Response Quality Tests
# ============================================================================


@pytest.mark.e2e
class TestResponseQuality:
    """Compare response quality across providers."""

    @pytest.fixture(autouse=True)
    def collect_results(self, request):
        """Collect all results for comparison."""
        self.results: list[LLMResponse] = []
        yield
        # Print summary after each test
        if self.results:
            print("\n" + "=" * 80)
            print("QUALITY COMPARISON")
            print("=" * 80)
            for r in self.results:
                status = "✓" if r.success else "✗"
                print(
                    f"{status} {r.provider:15} | {r.prompt_type:15} | {r.latency_ms:6.0f}ms | {r.response_length:4} chars"
                )
                if r.success:
                    print(f"   Content: {r.content[:100]}...")
                else:
                    print(f"   Error: {r.error}")
            print("=" * 80)

    def _test_all_providers(self, prompt_type: str) -> list[LLMResponse]:
        """Test all available providers for a given prompt type."""
        results = []
        for provider, config in PROVIDERS.items():
            # Skip if API key not available
            if config["skip_if_missing"] and not os.environ.get(
                config["skip_if_missing"]
            ):
                if provider != "opencode_cli":  # CLI doesn't need env var
                    continue
            response = call_llm(provider, prompt_type)
            results.append(response)
            self.results.append(response)
        return results

    def test_simple_prompt_quality(self):
        """Simple prompt should return 'hello' or similar."""
        results = self._test_all_providers("simple")
        for r in results:
            if r.success:
                # Should contain greeting-like content
                content_lower = r.content.lower()
                assert any(
                    word in content_lower
                    for word in ["hello", "hi", "hey", "greetings"]
                ), f"{r.provider} response doesn't look like a greeting: {r.content}"

    def test_json_format_quality(self):
        """JSON format prompt should return valid JSON."""
        import json

        results = self._test_all_providers("json_format")
        for r in results:
            if r.success:
                # Should be valid JSON
                try:
                    parsed = json.loads(r.content)
                    assert "greeting" in parsed or isinstance(parsed, dict), (
                        f"{r.provider} JSON missing expected key"
                    )
                except json.JSONDecodeError:
                    # Some models might wrap JSON in markdown
                    if "```json" in r.content or "```" in r.content:
                        # Extract JSON from markdown
                        import re

                        json_match = re.search(
                            r"```(?:json)?\s*([\s\S]*?)```", r.content
                        )
                        if json_match:
                            try:
                                json.loads(json_match.group(1).strip())
                            except json.JSONDecodeError:
                                pytest.fail(
                                    f"{r.provider} returned invalid JSON in markdown: {r.content}"
                                )
                    else:
                        pytest.fail(f"{r.provider} returned invalid JSON: {r.content}")

    def test_reasoning_quality(self):
        """Reasoning prompt should return correct answer."""
        results = self._test_all_providers("reasoning")
        for r in results:
            if r.success:
                # Should contain the number 5
                assert "5" in r.content, (
                    f"{r.provider} reasoning incorrect: {r.content}"
                )

    def test_creative_quality(self):
        """Creative prompt should return 3-line haiku."""
        results = self._test_all_providers("creative")
        for r in results:
            if r.success:
                # Should have multiple lines (haiku is 3 lines)
                lines = [l for l in r.content.strip().split("\n") if l.strip()]
                assert len(lines) >= 2, f"{r.provider} haiku too short: {r.content}"

    def test_structured_output_quality(self):
        """Structured output should follow format."""
        results = self._test_all_providers("structured")
        for r in results:
            if r.success:
                # Should have numbered list
                content = r.content
                # Check for any numbered list format
                has_numbering = any(
                    f"{i}." in content or f"{i})" in content for i in range(1, 4)
                )
                assert has_numbering, (
                    f"{r.provider} didn't return numbered list: {r.content}"
                )


# ============================================================================
# Compiler Task Quality Tests
# ============================================================================


@pytest.mark.e2e
class TestCompilerTaskQuality:
    """Test LLM quality on actual Compiler tasks."""

    @pytest.fixture(autouse=True)
    def collect_results(self):
        """Collect results for comparison."""
        self.results: list[LLMResponse] = []
        yield
        if self.results:
            print("\n" + "=" * 80)
            print("COMPILER TASK COMPARISON")
            print("=" * 80)
            for r in self.results:
                status = "✓" if r.success else "✗"
                print(
                    f"{status} {r.provider:15} | {r.latency_ms:6.0f}ms | {r.response_length:4} chars"
                )
                if r.success:
                    # Print key sections
                    content = r.content
                    if "<triage>" in content:
                        triage = content[
                            content.find("<triage>") : content.find("</triage>") + 10
                        ]
                        print(f"   Triage: {triage[:100]}...")
                    if "<synthesis>" in content:
                        synth = content[
                            content.find("<synthesis>") : content.find("</synthesis>")
                            + 12
                        ]
                        print(f"   Synthesis: {synth[:150]}...")
                else:
                    print(f"   Error: {r.error}")
            print("=" * 80)

    def test_compiler_task_format(self):
        """Compiler task should return properly formatted CoT."""
        results = []
        for provider, config in PROVIDERS.items():
            if config["skip_if_missing"] and not os.environ.get(
                config["skip_if_missing"]
            ):
                if provider != "opencode_cli":
                    continue
            response = call_llm(provider, "compiler_task")
            results.append(response)
            self.results.append(response)

        for r in results:
            if r.success:
                content = r.content
                # Check for expected XML tags
                assert "<triage>" in content, f"{r.provider} missing <triage>"
                assert "</triage>" in content, f"{r.provider} missing </triage>"
                assert "<cross_ref>" in content, f"{r.provider} missing <cross_ref>"
                assert "</cross_ref>" in content, f"{r.provider} missing </cross_ref>"
                assert "<synthesis>" in content, f"{r.provider} missing <synthesis>"
                assert "</synthesis>" in content, f"{r.provider} missing </synthesis>"

    def test_compiler_task_proposal_quality(self):
        """Compiler task should generate valid proposal structure."""
        for provider, config in PROVIDERS.items():
            if config["skip_if_missing"] and not os.environ.get(
                config["skip_if_missing"]
            ):
                if provider != "opencode_cli":
                    continue

            response = call_llm(provider, "compiler_task")
            self.results.append(response)

            if not response.success:
                continue

            content = response.content

            # Extract synthesis section
            if "<synthesis>" in content and "</synthesis>" in content:
                synth_start = content.find("<synthesis>") + len("<synthesis>")
                synth_end = content.find("</synthesis>")
                synthesis = content[synth_start:synth_end].strip()

                # Check proposal structure
                checks = [
                    (
                        "PROPOSAL:",
                        "## PROPOSAL:" in synthesis or "PROPOSAL:" in synthesis,
                    ),
                    ("Action:", "Action:" in synthesis),
                    ("Target:", "Target:" in synthesis),
                    ("Content:", "Content:" in synthesis),
                ]

                for check_name, check_result in checks:
                    assert check_result, f"{provider} synthesis missing {check_name}"


# ============================================================================
# Latency Comparison
# ============================================================================


@pytest.mark.e2e
class TestLatencyComparison:
    """Compare latency across providers."""

    @pytest.fixture(autouse=True)
    def collect_results(self):
        """Collect latency results."""
        self.latencies: dict[str, list[float]] = {}
        yield
        if self.latencies:
            print("\n" + "=" * 80)
            print("LATENCY COMPARISON (ms)")
            print("=" * 80)
            for provider, times in sorted(
                self.latencies.items(), key=lambda x: sum(x[1]) / len(x[1])
            ):
                avg = sum(times) / len(times)
                min_t = min(times)
                max_t = max(times)
                print(
                    f"{provider:15} | avg: {avg:7.0f} | min: {min_t:7.0f} | max: {max_t:7.0f}"
                )
            print("=" * 80)

    def test_latency_comparison(self):
        """Measure latency across providers with multiple calls."""
        num_calls = 3

        for provider, config in PROVIDERS.items():
            if config["skip_if_missing"] and not os.environ.get(
                config["skip_if_missing"]
            ):
                if provider != "opencode_cli":
                    continue

            self.latencies[provider] = []
            for i in range(num_calls):
                response = call_llm(provider, "simple")
                self.latencies[provider].append(response.latency_ms)
                if not response.success:
                    print(f"Warning: {provider} call {i + 1} failed: {response.error}")


# ============================================================================
# Cost Estimation
# ============================================================================


@pytest.mark.e2e
class TestCostEstimation:
    """Estimate costs for Compiler tasks."""

    # Approximate costs per 1M tokens (as of 2025)
    COSTS_PER_1M = {
        "openai": {"input": 0.25, "output": 2.00},  # gpt-5-mini
        "openrouter": {"input": 0.00, "output": 0.00},  # free model
        "ollama_cloud": {"input": 0.10, "output": 0.30},  # estimate
        "opencode_cli": {"input": 0.25, "output": 2.00},  # same as openai via CLI
    }

    def test_cost_estimation(self):
        """Estimate cost for a typical Compiler run."""
        # Typical Compiler run:
        # - Input: ~30 sessions × 3K tokens = 90K tokens
        # - Output: ~3 proposals × 1K tokens = 3K tokens

        input_tokens = 90_000
        output_tokens = 3_000

        print("\n" + "=" * 80)
        print("COST ESTIMATION PER EVOLVE RUN")
        print("=" * 80)
        print(f"Input tokens:  {input_tokens:,}")
        print(f"Output tokens: {output_tokens:,}")
        print("-" * 40)

        for provider, costs in self.COSTS_PER_1M.items():
            input_cost = (input_tokens / 1_000_000) * costs["input"]
            output_cost = (output_tokens / 1_000_000) * costs["output"]
            total = input_cost + output_cost
            print(
                f"{provider:15} | ${total:.4f} (in: ${input_cost:.4f}, out: ${output_cost:.4f})"
            )

        print("=" * 80)
        print("! Costs are approximate. OpenRouter free model has rate limits.")
        print("! OpenCode CLI cost depends on the underlying model used.")
