"""
Tests for the cloud quality evaluator and platform analytics.

Tests cover:
    - Execution recording and retrieval
    - Per-model stats aggregation (latency, success rate, tokens, P95)
    - Per-strategy stats (chain length comparison)
    - Improvement suggestions (model switch, chain simplification)
    - Platform analytics (user usage, leaderboard, platform stats)
"""

from __future__ import annotations

import json
import os
import time
from typing import Any

import pytest

from cloud.planner.quality_eval import (
    ModelStats,
    QualityEvaluator,
    StrategyStats,
    Suggestion,
)
from cloud.planner.analytics import (
    LeaderboardEntry,
    PlatformAnalytics,
    PlatformStats,
    UserUsage,
)


# ── Helpers ────────────────────────────────────────────────────────────


def _make_metrics(
    plan_id: str = "plan-001",
    steps: list[dict[str, Any]] | None = None,
    total_elapsed_ms: float = 1000.0,
    steps_completed: int = 1,
    steps_failed: int = 0,
) -> dict[str, Any]:
    """Build a metrics dict matching the format sent by the reporter."""
    if steps is None:
        steps = [
            {
                "step_id": "step_1",
                "elapsed_ms": 500.0,
                "tokens_used": 200,
                "success": True,
                "error_type": None,
                "model_name": "llama3",
                "backend": "ollama",
                "task_type": "summarization",
            }
        ]
    return {
        "plan_id": plan_id,
        "total_elapsed_ms": total_elapsed_ms,
        "steps_completed": steps_completed,
        "steps_failed": steps_failed,
        "steps": steps,
    }


# ══════════════════════════════════════════════════════════════════════
# QualityEvaluator Tests
# ══════════════════════════════════════════════════════════════════════


class TestQualityEvaluatorRecording:
    """Tests for recording and storing execution data."""

    @pytest.fixture
    def evaluator(self, tmp_path: Any) -> QualityEvaluator:
        return QualityEvaluator(data_dir=str(tmp_path / "quality"))

    def test_record_creates_file(self, evaluator: QualityEvaluator) -> None:
        """Recording an execution creates the JSONL file."""
        evaluator.record_execution("plan-001", _make_metrics())
        records = evaluator.storage.load_executions()
        assert len(records) == 1
        assert records[0]["plan_id"] == "plan-001"

    def test_record_multiple_plans(self, evaluator: QualityEvaluator) -> None:
        """Multiple recordings append correctly."""
        evaluator.record_execution("plan-001", _make_metrics(plan_id="plan-001"))
        evaluator.record_execution("plan-002", _make_metrics(plan_id="plan-002"))
        evaluator.record_execution("plan-003", _make_metrics(plan_id="plan-003"))

        records = evaluator.storage.load_executions()
        assert len(records) == 3
        plan_ids = {r["plan_id"] for r in records}
        assert plan_ids == {"plan-001", "plan-002", "plan-003"}

    def test_record_preserves_step_data(self, evaluator: QualityEvaluator) -> None:
        """Step-level data is preserved in storage."""
        steps = [
            {
                "step_id": "s1",
                "elapsed_ms": 100.0,
                "tokens_used": 50,
                "success": True,
                "model_name": "claude-sonnet",
                "backend": "claude",
                "task_type": "analysis",
            },
            {
                "step_id": "s2",
                "elapsed_ms": 5000.0,
                "tokens_used": 0,
                "success": False,
                "error_type": "timeout",
                "model_name": "gpt-4o",
                "backend": "gpt",
                "task_type": "generation",
            },
        ]
        evaluator.record_execution(
            "plan-x",
            _make_metrics(plan_id="plan-x", steps=steps, steps_failed=1),
        )
        records = evaluator.storage.load_executions()
        assert len(records[0]["steps"]) == 2
        assert records[0]["steps"][1]["error_type"] == "timeout"


class TestModelStats:
    """Tests for per-model statistics aggregation."""

    @pytest.fixture
    def evaluator(self, tmp_path: Any) -> QualityEvaluator:
        return QualityEvaluator(data_dir=str(tmp_path / "quality"))

    def test_no_data_returns_empty_stats(self, evaluator: QualityEvaluator) -> None:
        """A model with no data returns usage_count=0."""
        stats = evaluator.get_model_stats("nonexistent-model")
        assert stats.model_id == "nonexistent-model"
        assert stats.usage_count == 0
        assert stats.avg_latency_ms == 0.0

    def test_single_execution_stats(self, evaluator: QualityEvaluator) -> None:
        """Stats from a single execution are computed correctly."""
        evaluator.record_execution("p1", _make_metrics(
            steps=[{
                "step_id": "s1",
                "elapsed_ms": 1500.0,
                "tokens_used": 800,
                "success": True,
                "model_name": "claude-sonnet",
                "backend": "claude",
                "task_type": "analysis",
            }]
        ))

        stats = evaluator.get_model_stats("claude-sonnet")
        assert stats.usage_count == 1
        assert stats.avg_latency_ms == 1500.0
        assert stats.p95_latency_ms == 1500.0
        assert stats.success_rate == 1.0
        assert stats.avg_tokens_per_call == 800.0

    def test_multi_execution_averages(self, evaluator: QualityEvaluator) -> None:
        """Averages are computed across multiple executions."""
        for i, (ms, tok, ok) in enumerate([
            (100.0, 50, True),
            (200.0, 100, True),
            (300.0, 150, False),
            (400.0, 200, True),
        ]):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": ms,
                    "tokens_used": tok,
                    "success": ok,
                    "error_type": "timeout" if not ok else None,
                    "model_name": "llama3",
                    "backend": "ollama",
                    "task_type": "summarization",
                }]
            ))

        stats = evaluator.get_model_stats("llama3")
        assert stats.usage_count == 4
        assert stats.avg_latency_ms == 250.0  # (100+200+300+400)/4
        assert stats.success_rate == 0.75  # 3/4
        assert stats.avg_tokens_per_call == 125.0  # (50+100+150+200)/4
        assert stats.error_breakdown == {"timeout": 1}

    def test_p95_latency(self, evaluator: QualityEvaluator) -> None:
        """P95 latency is computed correctly from sorted values."""
        # Create 20 executions with latencies 100, 200, ..., 2000
        for i in range(20):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": (i + 1) * 100.0,
                    "tokens_used": 50,
                    "success": True,
                    "model_name": "fast-model",
                    "backend": "local",
                }]
            ))

        stats = evaluator.get_model_stats("fast-model")
        assert stats.usage_count == 20
        # P95 of [100, 200, ..., 2000] = 1900 (index 18 in 0-indexed sorted)
        assert stats.p95_latency_ms == 1900.0

    def test_model_filter_only_matches_target(
        self, evaluator: QualityEvaluator
    ) -> None:
        """Stats for model A don't include data from model B."""
        evaluator.record_execution("p1", _make_metrics(
            plan_id="p1",
            steps=[
                {"step_id": "s1", "elapsed_ms": 100.0, "tokens_used": 50,
                 "success": True, "model_name": "model_a", "backend": "x"},
                {"step_id": "s2", "elapsed_ms": 9999.0, "tokens_used": 5000,
                 "success": False, "model_name": "model_b", "backend": "y",
                 "error_type": "crash"},
            ]
        ))

        stats_a = evaluator.get_model_stats("model_a")
        assert stats_a.usage_count == 1
        assert stats_a.success_rate == 1.0
        assert stats_a.avg_latency_ms == 100.0

        stats_b = evaluator.get_model_stats("model_b")
        assert stats_b.usage_count == 1
        assert stats_b.success_rate == 0.0


class TestStrategyStats:
    """Tests for chain strategy statistics."""

    @pytest.fixture
    def evaluator(self, tmp_path: Any) -> QualityEvaluator:
        return QualityEvaluator(data_dir=str(tmp_path / "quality"))

    def test_no_matching_strategy(self, evaluator: QualityEvaluator) -> None:
        """A strategy with no data returns total_executions=0."""
        stats = evaluator.get_strategy_stats("5-step")
        assert stats.total_executions == 0

    def test_two_step_strategy(self, evaluator: QualityEvaluator) -> None:
        """2-step chains are correctly grouped and aggregated."""
        for i in range(5):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                total_elapsed_ms=1000.0 + i * 100,
                steps=[
                    {"step_id": "s1", "elapsed_ms": 500.0, "tokens_used": 100,
                     "success": True, "model_name": "a", "backend": "x"},
                    {"step_id": "s2", "elapsed_ms": 500.0 + i * 100, "tokens_used": 100,
                     "success": i != 3, "model_name": "b", "backend": "y",
                     "error_type": "timeout" if i == 3 else None},
                ],
                steps_completed=2 if i != 3 else 1,
                steps_failed=0 if i != 3 else 1,
            ))

        stats = evaluator.get_strategy_stats("2-step")
        assert stats.total_executions == 5
        assert stats.success_rate == 0.8  # 4/5 fully successful
        assert stats.avg_steps == 2.0
        assert stats.failure_rate_by_position[2] == 0.2  # Step 2 failed 1/5

    def test_invalid_strategy_format(self, evaluator: QualityEvaluator) -> None:
        """Invalid strategy_type string returns empty stats."""
        stats = evaluator.get_strategy_stats("invalid")
        assert stats.total_executions == 0


class TestSuggestions:
    """Tests for the improvement suggestion engine."""

    @pytest.fixture
    def evaluator(self, tmp_path: Any) -> QualityEvaluator:
        return QualityEvaluator(data_dir=str(tmp_path / "quality"))

    def test_no_data_no_suggestions(self, evaluator: QualityEvaluator) -> None:
        """No data means no suggestions."""
        suggestions = evaluator.suggest_improvements("plan-nonexistent")
        assert suggestions == []

    def test_slow_model_suggestion(self, evaluator: QualityEvaluator) -> None:
        """A consistently slow model triggers a model_switch suggestion."""
        # Record several slow executions for claude
        for i in range(5):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": 5000.0 + i * 100,  # All above 3000ms threshold
                    "tokens_used": 800,
                    "success": True,
                    "model_name": "claude-slow",
                    "backend": "claude",
                    "task_type": "summarization",
                }]
            ))

        suggestions = evaluator.suggest_improvements("p0")
        model_switches = [s for s in suggestions if s.category == "model_switch"]
        assert len(model_switches) >= 1
        assert "claude-slow" in model_switches[0].message
        assert model_switches[0].data["current_model"] == "claude-slow"

    def test_high_failure_rate_suggestion(self, evaluator: QualityEvaluator) -> None:
        """A model with high failure rate triggers a retry_strategy suggestion."""
        for i in range(6):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": 500.0,
                    "tokens_used": 100,
                    "success": i < 3,  # 50% failure rate
                    "error_type": "timeout" if i >= 3 else None,
                    "model_name": "flaky-model",
                    "backend": "remote",
                    "task_type": "analysis",
                }]
            ))

        suggestions = evaluator.suggest_improvements("p0")
        retry_suggestions = [s for s in suggestions if s.category == "retry_strategy"]
        assert len(retry_suggestions) >= 1
        assert "flaky-model" in retry_suggestions[0].message

    def test_chain_length_suggestion(self, evaluator: QualityEvaluator) -> None:
        """Longer chains with higher failure rates trigger chain_simplify suggestions."""
        # 2-step chains: all successful
        for i in range(5):
            evaluator.record_execution(f"short-{i}", _make_metrics(
                plan_id=f"short-{i}",
                total_elapsed_ms=500.0,
                steps=[
                    {"step_id": "s1", "elapsed_ms": 250.0, "tokens_used": 100,
                     "success": True, "model_name": "a", "backend": "x"},
                    {"step_id": "s2", "elapsed_ms": 250.0, "tokens_used": 100,
                     "success": True, "model_name": "b", "backend": "y"},
                ],
                steps_completed=2,
            ))

        # 3-step chains: 40% failure rate (much higher than 2-step 0%)
        for i in range(5):
            fail = i < 2  # 2 out of 5 fail
            evaluator.record_execution(f"long-{i}", _make_metrics(
                plan_id=f"long-{i}",
                total_elapsed_ms=1000.0,
                steps=[
                    {"step_id": "s1", "elapsed_ms": 300.0, "tokens_used": 100,
                     "success": True, "model_name": "a", "backend": "x"},
                    {"step_id": "s2", "elapsed_ms": 300.0, "tokens_used": 100,
                     "success": True, "model_name": "b", "backend": "y"},
                    {"step_id": "s3", "elapsed_ms": 400.0, "tokens_used": 100,
                     "success": not fail, "model_name": "c", "backend": "z",
                     "error_type": "model_error" if fail else None},
                ],
                steps_completed=3 if not fail else 2,
                steps_failed=0 if not fail else 1,
            ))

        suggestions = evaluator.suggest_improvements("long-0")
        chain_suggestions = [s for s in suggestions if s.category == "chain_simplify"]
        assert len(chain_suggestions) >= 1
        assert "3-step" in chain_suggestions[0].message

    def test_suggestions_sorted_by_confidence(self, evaluator: QualityEvaluator) -> None:
        """Suggestions are returned sorted by confidence descending."""
        # Create data that generates multiple suggestions
        for i in range(10):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": 8000.0,
                    "tokens_used": 100,
                    "success": i < 3,  # 70% failure AND slow
                    "error_type": "timeout" if i >= 3 else None,
                    "model_name": "bad-model",
                    "backend": "remote",
                    "task_type": "analysis",
                }]
            ))

        suggestions = evaluator.suggest_improvements("p0")
        if len(suggestions) >= 2:
            for i in range(len(suggestions) - 1):
                assert suggestions[i].confidence >= suggestions[i + 1].confidence


class TestQualitySummary:
    """Tests for the time-windowed quality summary."""

    @pytest.fixture
    def evaluator(self, tmp_path: Any) -> QualityEvaluator:
        return QualityEvaluator(data_dir=str(tmp_path / "quality"))

    def test_empty_summary(self, evaluator: QualityEvaluator) -> None:
        """No data returns zeroed summary."""
        summary = evaluator.get_quality_summary(hours=24)
        assert summary["total_plans"] == 0
        assert summary["success_rate"] == 0.0

    def test_summary_with_data(self, evaluator: QualityEvaluator) -> None:
        """Summary correctly aggregates recent data."""
        for i in range(4):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                total_elapsed_ms=1000.0 + i * 100,
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": 500.0,
                    "tokens_used": 100,
                    "success": i != 2,
                    "error_type": "crash" if i == 2 else None,
                    "model_name": "test-model",
                    "backend": "local",
                }],
                steps_failed=1 if i == 2 else 0,
            ))

        summary = evaluator.get_quality_summary(hours=24)
        assert summary["total_plans"] == 4
        assert summary["success_rate"] == 0.75  # 3 of 4 had no failures
        assert summary["model_health"]["test-model"] == 0.75
        assert summary["degraded_models"] == ["test-model"]  # 0.75 < 0.85 threshold

    def test_degraded_models_detected(self, evaluator: QualityEvaluator) -> None:
        """Models below 85% success rate appear in degraded_models."""
        for i in range(10):
            evaluator.record_execution(f"p{i}", _make_metrics(
                plan_id=f"p{i}",
                steps=[{
                    "step_id": "s1",
                    "elapsed_ms": 500.0,
                    "tokens_used": 100,
                    "success": i < 5,  # 50% success rate — definitely degraded
                    "error_type": "error" if i >= 5 else None,
                    "model_name": "bad-model",
                    "backend": "local",
                }],
                steps_failed=0 if i < 5 else 1,
            ))

        summary = evaluator.get_quality_summary(hours=24)
        assert "bad-model" in summary["degraded_models"]


# ══════════════════════════════════════════════════════════════════════
# Platform Analytics Tests
# ══════════════════════════════════════════════════════════════════════


class TestPlatformAnalytics:
    """Tests for the platform-wide analytics engine."""

    @pytest.fixture
    def analytics(self, tmp_path: Any) -> PlatformAnalytics:
        return PlatformAnalytics(data_dir=str(tmp_path / "analytics"))

    def test_record_and_get_user_usage(self, analytics: PlatformAnalytics) -> None:
        """User usage is correctly aggregated from events."""
        metrics1 = _make_metrics(
            plan_id="p1",
            steps=[
                {"step_id": "s1", "tokens_used": 100, "model_name": "llama3",
                 "success": True, "elapsed_ms": 200.0, "backend": "ollama"},
                {"step_id": "s2", "tokens_used": 200, "model_name": "claude",
                 "success": True, "elapsed_ms": 500.0, "backend": "claude"},
            ],
            total_elapsed_ms=700.0,
            steps_completed=2,
        )
        metrics2 = _make_metrics(
            plan_id="p2",
            steps=[
                {"step_id": "s1", "tokens_used": 300, "model_name": "llama3",
                 "success": True, "elapsed_ms": 150.0, "backend": "ollama"},
            ],
            total_elapsed_ms=150.0,
            steps_completed=1,
        )

        analytics.record_chain_execution("user-alice", metrics1)
        analytics.record_chain_execution("user-alice", metrics2)

        usage = analytics.get_user_usage("user-alice")
        assert usage.user_id == "user-alice"
        assert usage.total_chains == 2
        assert usage.total_tokens == 600  # 100 + 200 + 300
        assert usage.total_steps == 3
        assert set(usage.models_used) == {"claude", "llama3"}

    def test_user_not_found(self, analytics: PlatformAnalytics) -> None:
        """Unknown user returns empty usage."""
        usage = analytics.get_user_usage("ghost")
        assert usage.user_id == "ghost"
        assert usage.total_chains == 0
        assert usage.total_tokens == 0

    def test_user_isolation(self, analytics: PlatformAnalytics) -> None:
        """User A's data doesn't appear in User B's usage."""
        analytics.record_chain_execution("alice", _make_metrics(
            plan_id="pa",
            steps=[{"step_id": "s1", "tokens_used": 999, "model_name": "m",
                    "success": True, "elapsed_ms": 100.0, "backend": "x"}],
        ))
        analytics.record_chain_execution("bob", _make_metrics(
            plan_id="pb",
            steps=[{"step_id": "s1", "tokens_used": 1, "model_name": "m",
                    "success": True, "elapsed_ms": 100.0, "backend": "x"}],
        ))

        alice = analytics.get_user_usage("alice")
        bob = analytics.get_user_usage("bob")
        assert alice.total_tokens == 999
        assert bob.total_tokens == 1

    def test_platform_stats(self, analytics: PlatformAnalytics) -> None:
        """Platform-wide stats are computed correctly."""
        for user_id in ["alice", "bob", "charlie"]:
            analytics.record_chain_execution(user_id, _make_metrics(
                plan_id=f"p-{user_id}",
                total_elapsed_ms=1000.0,
                steps=[
                    {"step_id": "s1", "tokens_used": 100, "model_name": "llama3",
                     "success": True, "elapsed_ms": 500.0, "backend": "ollama",
                     "task_type": "summarization"},
                    {"step_id": "s2", "tokens_used": 200, "model_name": "claude",
                     "success": True, "elapsed_ms": 500.0, "backend": "claude",
                     "task_type": "analysis"},
                ],
                steps_completed=2,
            ))

        stats = analytics.get_platform_stats()
        assert stats.total_chains == 3
        assert stats.total_tokens == 900  # 300 per chain * 3
        assert stats.total_users == 3
        assert stats.error_rate == 0.0

        # Both models should appear in popular_models
        model_names = {m for m, _ in stats.popular_models}
        assert "llama3" in model_names
        assert "claude" in model_names

    def test_platform_stats_empty(self, analytics: PlatformAnalytics) -> None:
        """Empty platform returns zeroed stats."""
        stats = analytics.get_platform_stats()
        assert stats.total_chains == 0
        assert stats.total_users == 0

    def test_model_leaderboard(self, analytics: PlatformAnalytics) -> None:
        """Leaderboard ranks models by success rate per task type."""
        # Model A: fast and reliable for summarization
        for i in range(5):
            analytics.record_chain_execution("user", _make_metrics(
                plan_id=f"pa-{i}",
                steps=[{"step_id": "s1", "elapsed_ms": 100.0, "tokens_used": 50,
                        "success": True, "model_name": "fast-model",
                        "backend": "local", "task_type": "summarization"}],
            ))

        # Model B: slow and flaky for summarization
        for i in range(5):
            analytics.record_chain_execution("user", _make_metrics(
                plan_id=f"pb-{i}",
                steps=[{"step_id": "s1", "elapsed_ms": 3000.0, "tokens_used": 500,
                        "success": i < 3, "model_name": "slow-model",
                        "backend": "remote", "task_type": "summarization",
                        "error_type": "timeout" if i >= 3 else None}],
            ))

        leaderboard = analytics.get_model_leaderboard()
        assert len(leaderboard) >= 2

        # fast-model should rank above slow-model for summarization
        summarization_entries = [e for e in leaderboard if e.task_type == "summarization"]
        assert summarization_entries[0].model_id == "fast-model"
        assert summarization_entries[0].success_rate == 1.0
        assert summarization_entries[1].model_id == "slow-model"
        assert summarization_entries[1].success_rate == 0.6

    def test_leaderboard_empty(self, analytics: PlatformAnalytics) -> None:
        """Empty platform returns empty leaderboard."""
        leaderboard = analytics.get_model_leaderboard()
        assert leaderboard == []

    def test_first_seen_last_seen(self, analytics: PlatformAnalytics) -> None:
        """User's first_seen and last_seen timestamps are tracked."""
        analytics.record_chain_execution("alice", _make_metrics(plan_id="p1"))
        # Small delay to ensure different timestamps
        analytics.record_chain_execution("alice", _make_metrics(plan_id="p2"))

        usage = analytics.get_user_usage("alice")
        assert usage.first_seen > 0
        assert usage.last_seen >= usage.first_seen

    def test_error_rate_computed(self, analytics: PlatformAnalytics) -> None:
        """Platform error rate is computed from step-level failures."""
        # 2 successful + 2 failed steps
        analytics.record_chain_execution("user", _make_metrics(
            plan_id="p1",
            steps=[
                {"step_id": "s1", "elapsed_ms": 100.0, "tokens_used": 50,
                 "success": True, "model_name": "a", "backend": "x"},
                {"step_id": "s2", "elapsed_ms": 100.0, "tokens_used": 50,
                 "success": False, "model_name": "b", "backend": "y",
                 "error_type": "crash"},
            ],
            steps_completed=1,
            steps_failed=1,
        ))
        analytics.record_chain_execution("user", _make_metrics(
            plan_id="p2",
            steps=[
                {"step_id": "s1", "elapsed_ms": 100.0, "tokens_used": 50,
                 "success": True, "model_name": "a", "backend": "x"},
                {"step_id": "s2", "elapsed_ms": 100.0, "tokens_used": 50,
                 "success": False, "model_name": "b", "backend": "y",
                 "error_type": "timeout"},
            ],
            steps_completed=1,
            steps_failed=1,
        ))

        stats = analytics.get_platform_stats()
        assert stats.error_rate == 0.5  # 2 failures out of 4 steps
