"""Tests for the P2 evaluation loop."""
from __future__ import annotations

import asyncio
from dataclasses import asdict
from datetime import UTC, datetime, timedelta

import pytest

from logos_adk.agent import Agent
from logos_adk.learning import (
    ContinuousEvalRunner,
    EvaluationLoopEngine,
    PromptRegistry,
    ReflectionMemory,
    SelfLearningLoop,
    SelfLearningLoopPolicy,
    SQLiteLearningStore,
)
from logos_adk.learning.evaluation.models import ContinuousEvalJob
from logos_adk.learning.models import FeedbackRecord, OutcomeRecord, RunRecord
from logos_adk.providers.mock import MockProvider
from logos_adk.quality_history import QualityExecutionManager, QualityRunRecord, SQLiteQualityHistoryStore
from logos_adk.testing_framework import JudgeResult, LLMJudge
from logos_adk.types import Response, Usage


class EchoSystemPromptProvider(MockProvider):
    async def generate(self, messages, tools=None, output_schema=None, **kwargs):
        system_prompt = next((message.content for message in messages if message.role == "system"), "")
        return Response(
            content=system_prompt,
            tool_calls=[],
            usage=Usage(input_tokens=20, output_tokens=10),
            raw={},
        )


class CandidateAwareProvider(MockProvider):
    async def generate(self, messages, tools=None, output_schema=None, **kwargs):
        system_prompt = next((message.content for message in messages if message.role == "system"), "")
        content = "candidate-good" if "candidate-good" in system_prompt else "candidate-bad"
        return Response(
            content=content,
            tool_calls=[],
            usage=Usage(input_tokens=20, output_tokens=8),
            raw={},
        )


def _make_run(
    *,
    run_id: str,
    trace_id: str,
    created_at: datetime,
    input_text: str,
    output_text: str,
    workspace_id: str = "acme",
    agent_name: str = "bot",
    prompt_version: str = "v1",
    model: str = "gpt-4o-mini",
    latency_ms: float | None = None,
    tool_calls: list[dict] | None = None,
    metadata: dict | None = None,
) -> RunRecord:
    payload = dict(metadata or {})
    if latency_ms is not None:
        payload["latency_ms"] = latency_ms
    return RunRecord(
        id=run_id,
        trace_id=trace_id,
        request_id=None,
        agent_name=agent_name,
        input_text=input_text,
        output_text=output_text,
        created_at=created_at.isoformat(),
        session_id="s1",
        workspace_id=workspace_id,
        prompt_version=prompt_version,
        model=model,
        usage={"input_tokens": 100, "output_tokens": 50},
        tool_calls=list(tool_calls or []),
        metadata=payload,
    )


def _feedback(
    *,
    feedback_id: str,
    trace_id: str,
    run_id: str,
    kind: str,
    created_at: datetime,
    corrected_output: str | None = None,
    expected_output: str | None = None,
    rating: float | None = None,
    failure_types: list[str] | None = None,
) -> FeedbackRecord:
    return FeedbackRecord(
        id=feedback_id,
        kind=kind,  # type: ignore[arg-type]
        run_id=run_id,
        trace_id=trace_id,
        created_at=created_at.isoformat(),
        workspace_id="acme",
        agent_name="bot",
        prompt_version="v1",
        model="gpt-4o-mini",
        corrected_output=corrected_output,
        expected_output=expected_output,
        rating=rating,
        failure_types=list(failure_types or []),  # type: ignore[arg-type]
    )


def _outcome(
    *,
    outcome_id: str,
    trace_id: str,
    run_id: str,
    kind: str,
    value,
    created_at: datetime,
) -> OutcomeRecord:
    return OutcomeRecord(
        id=outcome_id,
        kind=kind,  # type: ignore[arg-type]
        run_id=run_id,
        trace_id=trace_id,
        created_at=created_at.isoformat(),
        workspace_id="acme",
        agent_name="bot",
        prompt_version="v1",
        model="gpt-4o-mini",
        value=value,
    )


@pytest.mark.asyncio
async def test_continuous_eval_runner_builds_prod_regression_and_persists_quality_run(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)

    created_at = datetime.now(UTC) - timedelta(minutes=10)
    learning_store.create_run(
        _make_run(
            run_id="run-1",
            trace_id="trace-1",
            created_at=created_at,
            input_text="Write a short reply",
            output_text="bad answer",
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-1",
            trace_id="trace-1",
            run_id="run-1",
            kind="edited_answer",
            created_at=created_at,
            corrected_output="good answer",
            failure_types=["hallucination"],
        )
    )

    dataset = engine.build_regression_dataset(workspace_id="acme", agent_name="bot", limit=10)
    assert len(dataset.examples) == 1
    assert dataset.examples[0].expected == "good answer"

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager)
    runner.register_regression_job(
        "prod-regression",
        agent=agent,
        workspace_id="acme",
        agent_name="bot",
        interval_seconds=0.05,
        limit=10,
    )

    record = await runner.run_job_once("prod-regression")
    assert record.status == "completed"
    assert record.summary == {"passed": 1, "failed": 0, "total": 1}

    await runner.start()
    await asyncio.sleep(0.12)
    await runner.stop()
    assert len(quality_manager.list_runs(agent_name="bot", kind="regression", limit=10)) >= 2


def test_continuous_eval_runner_restores_persisted_jobs(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    path = str(tmp_path / "continuous-eval.db")

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    runner.register_regression_job(
        "prod-regression",
        agent=agent,
        workspace_id="acme",
        agent_name="bot",
        interval_seconds=60.0,
        limit=10,
        metadata={"created_via": "test"},
    )

    restored = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    jobs = restored.restore_jobs()

    assert len(jobs) == 1
    assert jobs[0]["name"] == "prod-regression"
    assert jobs[0]["metadata"]["created_via"] == "test"


def test_continuous_eval_runner_restore_jobs_filters_by_agent_name_and_restores_mock_judge(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    path = str(tmp_path / "continuous-eval.db")

    bot = Agent("bot", model=EchoSystemPromptProvider(), system="bot answer")
    other = Agent("other", model=EchoSystemPromptProvider(), system="other answer")
    runner = ContinuousEvalRunner(engine, quality_manager, path=path, agent=bot, agent_name="bot")
    runner.register_regression_job(
        "bot-regression",
        agent=bot,
        workspace_id="acme",
        agent_name="bot",
        metadata={
            "created_via": "test",
            "judge_kind": "mock",
            "mock_response": '{"score": 0.9, "rationale": "ok"}',
            "pass_threshold": 0.6,
        },
    )
    runner.register_regression_job(
        "other-regression",
        agent=other,
        workspace_id="acme",
        agent_name="other",
        metadata={"created_via": "other"},
    )

    restored = ContinuousEvalRunner(engine, quality_manager, path=path, agent=bot, agent_name="bot")
    jobs = restored.restore_jobs()
    jobs_again = restored.restore_jobs()

    assert [job["name"] for job in jobs] == ["bot-regression"]
    assert jobs[0]["has_judge"] is True
    assert jobs[0]["restored"] is True
    assert jobs[0]["restore_count"] == 1
    assert jobs_again == jobs


@pytest.mark.asyncio
async def test_continuous_eval_runner_remove_job_cancels_task_and_deletes_persistence(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    path = str(tmp_path / "continuous-eval.db")

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    job = runner.register_regression_job(
        "prod-regression",
        agent=agent,
        workspace_id="acme",
        agent_name="bot",
    )
    job.task = asyncio.create_task(asyncio.sleep(10))

    removed = await runner.remove_job("prod-regression")
    missing = await runner.remove_job("missing-job")
    restored = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")

    assert removed is not None
    assert removed["name"] == "prod-regression"
    assert removed["running"] is False
    assert missing is None
    assert restored.restore_jobs() == []


def test_continuous_eval_runner_resolve_incident_auto_acknowledges_and_records_notes(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    path = str(tmp_path / "continuous-eval.db")

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    job = runner.register_regression_job(
        "prod-regression",
        agent=agent,
        workspace_id="acme",
        agent_name="bot",
    )
    incident = runner._create_incident(
        job=job,
        kind="regression_threshold_breach",
        severity="warning",
        message="Pass rate dropped below threshold.",
        run_id="run-1",
        prompt_name="bot",
        metadata={"drop": 0.42},
    )

    resolved = runner.resolve_incident(incident.id, actor="ops", note="handled")

    assert resolved is not None
    assert resolved["status"] == "resolved"
    assert resolved["acknowledged_by"] == "ops"
    assert resolved["resolved_by"] == "ops"
    assert resolved["metadata"]["operator_notes"][0]["status"] == "resolved"
    assert resolved["metadata"]["operator_notes"][0]["note"] == "handled"
    assert runner.resolve_incident("missing-incident", actor="ops", note="missing") is None


def test_continuous_eval_runner_restore_judge_requires_mock_kind_and_agent(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, agent=agent, agent_name="bot")
    no_agent_runner = ContinuousEvalRunner(engine, quality_manager)

    mock_judge = runner._restore_judge(
        {"judge_kind": "mock", "mock_response": '{"score": 1.0}', "pass_threshold": 0.75}
    )
    non_mock = runner._restore_judge({"judge_kind": "human"})
    no_agent = no_agent_runner._restore_judge({"judge_kind": "mock"})

    assert mock_judge is not None
    assert isinstance(mock_judge, LLMJudge)
    assert mock_judge.pass_threshold == 0.75
    assert non_mock is None
    assert no_agent is None


@pytest.mark.asyncio
async def test_self_learning_loop_registers_eval_job_and_runner_can_auto_rollback(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    registry = PromptRegistry(str(tmp_path / "prompts.db"))
    reflection_memory = ReflectionMemory(path=str(tmp_path / "reflection.db"))
    runner = ContinuousEvalRunner(
        engine,
        quality_manager,
        path=str(tmp_path / "continuous-eval.db"),
        prompt_registry=registry,
    )

    created_at = datetime.now(UTC) - timedelta(minutes=10)
    learning_store.create_run(
        _make_run(
            run_id="run-sl",
            trace_id="trace-sl",
            created_at=created_at,
            input_text="Write a short reply",
            output_text="bad answer",
            metadata={"prompt_name": "bot"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-sl",
            trace_id="trace-sl",
            run_id="run-sl",
            kind="edited_answer",
            created_at=created_at,
            corrected_output="candidate-good",
            failure_types=["hallucination"],
        )
    )

    registry.ensure_prompt("bot")
    baseline = registry.create_version("bot", content="baseline prompt")
    registry.promote("bot", version=baseline.version, workspace_id="acme")

    agent = Agent("bot", model=CandidateAwareProvider(), system="fallback")
    runner.default_agent = agent
    runner.default_agent_name = "bot"
    loop = SelfLearningLoop(
        learning_store,
        registry,
        reflection_memory=reflection_memory,
        continuous_eval_runner=runner,
    )
    result = await loop.run_once(
        agent,
        prompt_name="bot",
        workspace_id="acme",
        policy=SelfLearningLoopPolicy(
            min_sample_size=1,
            min_improvement=0.01,
            allow_auto_promote=True,
            rollback_min_pass_rate=1.1,
            rollback_min_total_cases=1,
            continuous_eval_interval_seconds=60.0,
        ),
    )

    assert result.action is not None
    assert result.action.kind == "promote"
    job_name = result.action.metadata["continuous_eval_job"]
    assert job_name is not None
    assert runner.get_job(job_name) is not None

    record = await runner.run_job_once(job_name)
    resolved = registry.resolve("bot", workspace_id="acme")
    assert resolved is not None
    assert resolved.version == baseline.version
    refreshed = quality_manager.get(record.id)
    assert refreshed is not None
    assert "auto_rollback" in (refreshed.reports or {})


@pytest.mark.asyncio
async def test_self_learning_loop_experiment_can_be_auto_disabled_after_sustained_regression(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    registry = PromptRegistry(str(tmp_path / "prompts.db"))
    reflection_memory = ReflectionMemory(path=str(tmp_path / "reflection.db"))
    runner = ContinuousEvalRunner(
        engine,
        quality_manager,
        path=str(tmp_path / "continuous-eval.db"),
        prompt_registry=registry,
    )

    created_at = datetime.now(UTC) - timedelta(minutes=10)
    learning_store.create_run(
        _make_run(
            run_id="run-exp",
            trace_id="trace-exp",
            created_at=created_at,
            input_text="Write a short reply",
            output_text="bad answer",
            metadata={"prompt_name": "bot"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-exp",
            trace_id="trace-exp",
            run_id="run-exp",
            kind="edited_answer",
            created_at=created_at,
            corrected_output="candidate-good",
            failure_types=["hallucination"],
        )
    )

    registry.ensure_prompt("bot")
    baseline = registry.create_version("bot", content="baseline prompt")
    registry.promote("bot", version=baseline.version, workspace_id="acme")

    agent = Agent("bot", model=CandidateAwareProvider(), system="fallback")
    runner.default_agent = agent
    runner.default_agent_name = "bot"
    loop = SelfLearningLoop(
        learning_store,
        registry,
        reflection_memory=reflection_memory,
        continuous_eval_runner=runner,
    )
    result = await loop.run_once(
        agent,
        prompt_name="bot",
        workspace_id="acme",
        policy=SelfLearningLoopPolicy(
            min_sample_size=1,
            min_improvement=0.01,
            allow_auto_promote=False,
            rollback_min_pass_rate=1.1,
            rollback_min_total_cases=1,
            disable_experiment_after=2,
            continuous_eval_interval_seconds=60.0,
        ),
    )

    assert result.action is not None
    assert result.action.kind == "experiment"
    job_name = result.action.metadata["continuous_eval_job"]
    experiment_id = result.experiment["id"]
    await runner.run_job_once(job_name)
    experiment_after_one = registry.get_experiment(experiment_id)
    assert experiment_after_one is not None
    assert experiment_after_one.status == "active"
    await runner.run_job_once(job_name)
    experiment_after_two = registry.get_experiment(experiment_id)
    assert experiment_after_two is not None
    assert experiment_after_two.status == "disabled"


def test_judge_disagreement_tracking_detects_human_vs_judge_mismatch(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    engine = EvaluationLoopEngine(learning_store)

    created_at = datetime.now(UTC) - timedelta(hours=2)
    learning_store.create_run(
        _make_run(
            run_id="run-judge",
            trace_id="trace-judge",
            created_at=created_at,
            input_text="Answer the customer",
            output_text="bad answer",
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-judge",
            trace_id="trace-judge",
            run_id="run-judge",
            kind="thumbs_down",
            created_at=created_at,
            failure_types=["style_miss"],
        )
    )

    cases = engine.build_judge_cases_from_prod(workspace_id="acme", agent_name="bot", limit=10)
    assert cases == []

    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-judge-correction",
            trace_id="trace-judge",
            run_id="run-judge",
            kind="edited_answer",
            created_at=created_at,
            corrected_output="approved answer",
            failure_types=["style_miss"],
        )
    )
    cases = engine.build_judge_cases_from_prod(workspace_id="acme", agent_name="bot", limit=10)
    assert cases[0].name == "run-judge"

    disagreements = engine.track_judge_disagreements(
        [JudgeResult(name="run-judge", score=0.95, passed=True, rationale="looks good", answer="bad answer")],
        workspace_id="acme",
        agent_name="bot",
        limit=10,
    )
    assert len(disagreements) == 1
    assert disagreements[0].judge_passed is True
    assert disagreements[0].human_passed is False
    assert disagreements[0].delta >= 0.35


def test_learning_scorecard_computes_weekly_deltas_from_runs_and_feedback(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    engine = EvaluationLoopEngine(learning_store)
    now = datetime.now(UTC)

    previous_at = now - timedelta(days=10)
    current_at = now - timedelta(days=2)

    learning_store.create_run(
        _make_run(
            run_id="run-prev",
            trace_id="trace-prev",
            created_at=previous_at,
            input_text="Prev",
            output_text="prev answer",
            latency_ms=1400,
            tool_calls=[{"id": "1", "name": "search", "arguments": {}}],
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-prev",
            trace_id="trace-prev",
            run_id="run-prev",
            kind="thumbs_down",
            created_at=previous_at,
            failure_types=["bad_retrieval", "wrong_tool"],
        )
    )

    learning_store.create_run(
        _make_run(
            run_id="run-current",
            trace_id="trace-current",
            created_at=current_at,
            input_text="Current",
            output_text="current answer",
            latency_ms=400,
            tool_calls=[{"id": "1", "name": "search", "arguments": {}}],
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-current",
            trace_id="trace-current",
            run_id="run-current",
            kind="thumbs_up",
            created_at=current_at,
        )
    )
    learning_store.create_outcome(
        _outcome(
            outcome_id="out-current",
            trace_id="trace-current",
            run_id="run-current",
            kind="conversion",
            value=True,
            created_at=current_at,
        )
    )

    scorecard = engine.build_scorecard(
        workspace_id="acme",
        agent_name="bot",
        window_days=7,
        end_at=now,
        limit=50,
    )
    assert scorecard.quality.delta > 0
    assert scorecard.latency.delta < 0
    assert scorecard.retrieval_relevance.delta > 0
    assert scorecard.tool_success.delta > 0
    assert scorecard.counts["current"]["runs"] == 1
    assert scorecard.counts["previous"]["runs"] == 1


@pytest.mark.asyncio
async def test_champion_challenger_framework_compares_offline_and_live_slices(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    engine = EvaluationLoopEngine(learning_store)
    now = datetime.now(UTC)

    learning_store.create_run(
        _make_run(
            run_id="run-offline",
            trace_id="trace-offline",
            created_at=now - timedelta(hours=1),
            input_text="Need a better answer",
            output_text="bad",
            metadata={"prompt_variant": "baseline"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-offline",
            trace_id="trace-offline",
            run_id="run-offline",
            kind="edited_answer",
            created_at=now - timedelta(hours=1),
            corrected_output="challenger",
            failure_types=["hallucination"],
        )
    )

    learning_store.create_run(
        _make_run(
            run_id="run-live-base",
            trace_id="trace-live-base",
            created_at=now - timedelta(minutes=50),
            input_text="live base",
            output_text="baseline",
            metadata={"prompt_variant": "baseline"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-live-base",
            trace_id="trace-live-base",
            run_id="run-live-base",
            kind="thumbs_down",
            created_at=now - timedelta(minutes=50),
            failure_types=["style_miss"],
        )
    )

    learning_store.create_run(
        _make_run(
            run_id="run-live-challenger",
            trace_id="trace-live-challenger",
            created_at=now - timedelta(minutes=40),
            input_text="live challenger",
            output_text="challenger",
            metadata={"prompt_variant": "challenger-v2"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-live-challenger",
            trace_id="trace-live-challenger",
            run_id="run-live-challenger",
            kind="thumbs_up",
            created_at=now - timedelta(minutes=40),
        )
    )
    learning_store.create_outcome(
        _outcome(
            outcome_id="out-live-challenger",
            trace_id="trace-live-challenger",
            run_id="run-live-challenger",
            kind="conversion",
            value=True,
            created_at=now - timedelta(minutes=40),
        )
    )

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="baseline")
    report = await engine.compare_champion_challenger(
        agent,
        champion_prompt="baseline",
        challenger_prompt="challenger",
        workspace_id="acme",
        agent_name="bot",
        champion_filter={"prompt_variant": "baseline"},
        challenger_filter={"prompt_variant": "challenger-v2"},
        limit=20,
    )

    assert report.champion["offline"]["pass_rate"] == 0.0
    assert report.challenger["offline"]["pass_rate"] == 1.0
    assert report.challenger["live"]["reward_score"] > report.champion["live"]["reward_score"]
    assert report.recommendation == "challenger"


@pytest.mark.asyncio
async def test_champion_challenger_supports_learned_reward_scorer_mode(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    engine = EvaluationLoopEngine(learning_store)
    now = datetime.now(UTC)

    learning_store.create_run(
        _make_run(
            run_id="run-offline-learned",
            trace_id="trace-offline-learned",
            created_at=now - timedelta(hours=1),
            input_text="Need a better answer",
            output_text="bad",
            metadata={"prompt_variant": "baseline"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-offline-learned",
            trace_id="trace-offline-learned",
            run_id="run-offline-learned",
            kind="edited_answer",
            created_at=now - timedelta(hours=1),
            corrected_output="challenger",
            failure_types=["hallucination"],
        )
    )
    learning_store.create_run(
        _make_run(
            run_id="run-live-base-learned",
            trace_id="trace-live-base-learned",
            created_at=now - timedelta(minutes=50),
            input_text="live base",
            output_text="baseline",
            metadata={"prompt_variant": "baseline"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-live-base-learned",
            trace_id="trace-live-base-learned",
            run_id="run-live-base-learned",
            kind="thumbs_down",
            created_at=now - timedelta(minutes=50),
            failure_types=["style_miss"],
        )
    )
    learning_store.create_run(
        _make_run(
            run_id="run-live-challenger-learned",
            trace_id="trace-live-challenger-learned",
            created_at=now - timedelta(minutes=40),
            input_text="live challenger",
            output_text="challenger",
            metadata={"prompt_variant": "challenger-v2"},
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-live-challenger-learned",
            trace_id="trace-live-challenger-learned",
            run_id="run-live-challenger-learned",
            kind="thumbs_up",
            created_at=now - timedelta(minutes=40),
        )
    )
    learning_store.create_outcome(
        _outcome(
            outcome_id="out-live-challenger-learned",
            trace_id="trace-live-challenger-learned",
            run_id="run-live-challenger-learned",
            kind="conversion",
            value=True,
            created_at=now - timedelta(minutes=40),
        )
    )

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="baseline")
    report = await engine.compare_champion_challenger(
        agent,
        champion_prompt="baseline",
        challenger_prompt="challenger",
        workspace_id="acme",
        agent_name="bot",
        champion_filter={"prompt_variant": "baseline"},
        challenger_filter={"prompt_variant": "challenger-v2"},
        limit=20,
        reward_scorer_mode="learned",
        reward_training_limit=50,
    )

    assert report.reward_scorer_mode == "learned"
    assert report.challenger["live"]["reward_score"] > report.champion["live"]["reward_score"]


@pytest.mark.asyncio
async def test_run_judge_snapshot_includes_disagreement_count(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    engine = EvaluationLoopEngine(learning_store)
    created_at = datetime.now(UTC) - timedelta(minutes=30)
    learning_store.create_run(
        _make_run(
            run_id="run-judge-snapshot",
            trace_id="trace-judge-snapshot",
            created_at=created_at,
            input_text="Need the best answer",
            output_text="bad",
        )
    )
    learning_store.create_feedback(
        _feedback(
            feedback_id="fb-judge-snapshot",
            trace_id="trace-judge-snapshot",
            run_id="run-judge-snapshot",
            kind="edited_answer",
            created_at=created_at,
            corrected_output="ideal",
            failure_types=["hallucination"],
        )
    )

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="ideal")
    judge = LLMJudge(lambda answer: (0.2, "human preferred correction"))
    payload = await engine.run_judge_snapshot(
        agent,
        judge=judge,
        workspace_id="acme",
        agent_name="bot",
        limit=10,
    )
    assert payload["summary"]["total"] == 1
    assert payload["summary"]["judge_disagreements"] == 0
    assert payload["rows"][0]["passed"] is False
    assert asdict(engine.build_scorecard(workspace_id="acme", agent_name="bot", limit=10))


def test_continuous_eval_runner_trend_delta_report_tracks_window_deltas_and_failures(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    path = str(tmp_path / "continuous-eval.db")

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    job = runner.register_regression_job(
        "prod-regression",
        agent=agent,
        workspace_id="acme",
        agent_name="bot",
    )
    now = datetime.now(UTC)
    quality_manager.store.create_run(
        QualityRunRecord(
            id="run-prev",
            agent_name="bot",
            kind="regression",
            status="completed",
            created_at=(now - timedelta(days=10)).isoformat(),
            summary={"passed": 2, "failed": 0, "total": 2},
        )
    )
    quality_manager.store.create_run(
        QualityRunRecord(
            id="run-current",
            agent_name="bot",
            kind="regression",
            status="completed",
            created_at=(now - timedelta(days=1)).isoformat(),
            summary={"passed": 1, "failed": 1, "total": 2},
        )
    )
    quality_manager.store.create_run(
        QualityRunRecord(
            id="run-current-failed",
            agent_name="bot",
            kind="regression",
            status="failed",
            created_at=(now - timedelta(hours=6)).isoformat(),
            error="judge timeout",
        )
    )

    current_incident = runner._create_incident(
        job=job,
        kind="prompt_auto_rollback",
        severity="critical",
        message="rolled back",
        run_id="run-current",
        prompt_name="bot",
    )
    previous_incident = runner._create_incident(
        job=job,
        kind="experiment_auto_disabled",
        severity="critical",
        message="disabled",
        run_id="run-prev",
        prompt_name="bot",
        experiment_id="exp-1",
    )
    with runner._connect() as connection:
        connection.execute(
            "UPDATE continuous_eval_incidents SET created_at = ? WHERE id = ?",
            ((now - timedelta(days=1)).isoformat(), current_incident.id),
        )
        connection.execute(
            "UPDATE continuous_eval_incidents SET created_at = ? WHERE id = ?",
            ((now - timedelta(days=10)).isoformat(), previous_incident.id),
        )
        connection.commit()

    report = runner.trend_delta_report(workspace_id="acme", job_name="prod-regression", window_days=7)

    assert report.pass_rate.current == 0.5
    assert report.pass_rate.previous == 1.0
    assert report.pass_rate.delta == -0.5
    assert report.incidents.current == 1.0
    assert report.incidents.previous == 1.0
    assert report.rollbacks.current == 1.0
    assert report.rollbacks.previous == 0.0
    assert report.experiment_disables.current == 0.0
    assert report.experiment_disables.previous == 1.0
    assert report.run_failures.current == 1.0
    assert report.run_failures.previous == 0.0
    assert report.counts == {
        "current_runs": 2,
        "previous_runs": 1,
        "current_incidents": 1,
        "previous_incidents": 1,
    }


def test_continuous_eval_runner_incident_snapshot_groups_statuses_and_restored_jobs(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)
    path = str(tmp_path / "continuous-eval.db")

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    runner.register_regression_job(
        "prod-regression",
        agent=agent,
        workspace_id="acme",
        agent_name="bot",
    )

    restored = ContinuousEvalRunner(engine, quality_manager, path=path, agent=agent, agent_name="bot")
    restored.restore_jobs()
    job = restored.get_job("prod-regression")
    assert job is not None

    open_incident = restored._create_incident(
        job=job,
        kind="regression_threshold_breach",
        severity="warning",
        message="open",
        prompt_name="bot",
    )
    acknowledged_incident = restored._create_incident(
        job=job,
        kind="prompt_auto_rollback",
        severity="critical",
        message="ack",
        prompt_name="bot",
    )
    resolved_incident = restored._create_incident(
        job=job,
        kind="experiment_auto_disabled",
        severity="critical",
        message="resolved",
        prompt_name="bot",
        experiment_id="exp-1",
    )
    restored.acknowledge_incident(acknowledged_incident.id, actor="ops", note="triaged")
    restored.resolve_incident(resolved_incident.id, actor="ops", note="handled")
    now = datetime.now(UTC)
    with restored._connect() as connection:
        connection.execute(
            "UPDATE continuous_eval_incidents SET created_at = ? WHERE id = ?",
            ((now - timedelta(hours=3)).isoformat(), open_incident.id),
        )
        connection.execute(
            "UPDATE continuous_eval_incidents SET created_at = ? WHERE id = ?",
            ((now - timedelta(hours=2)).isoformat(), acknowledged_incident.id),
        )
        connection.execute(
            "UPDATE continuous_eval_incidents SET created_at = ? WHERE id = ?",
            ((now - timedelta(hours=1)).isoformat(), resolved_incident.id),
        )
        connection.commit()

    snapshot = restored.incident_snapshot(workspace_id="acme", job_name="prod-regression", limit=2)
    resolved_only = restored.incident_snapshot(
        workspace_id="acme",
        job_name="prod-regression",
        status="resolved",
        limit=10,
    )

    assert snapshot.summary["total_incidents"] == 3
    assert snapshot.summary["open_incidents"] == 1
    assert snapshot.summary["acknowledged_incidents"] == 1
    assert snapshot.summary["resolved_incidents"] == 1
    assert snapshot.summary["by_kind"]["prompt_auto_rollback"] == 1
    assert snapshot.summary["by_severity"]["critical"] == 2
    assert snapshot.summary["restored_jobs"] == 1
    assert [item["id"] for item in snapshot.latest_incidents] == [
        resolved_incident.id,
        acknowledged_incident.id,
    ]
    assert snapshot.affected_prompts == [{"prompt_name": "bot", "incident_count": 3}]
    assert snapshot.affected_jobs == [{"job_name": "prod-regression", "incident_count": 3}]
    assert snapshot.restored_jobs[0]["name"] == "prod-regression"
    assert resolved_only.summary["total_incidents"] == 1
    assert resolved_only.latest_incidents[0]["id"] == resolved_incident.id


@pytest.mark.asyncio
async def test_continuous_eval_runner_run_job_once_marks_failed_when_judge_missing(tmp_path):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    quality_store = SQLiteQualityHistoryStore(str(tmp_path / "quality.db"))
    quality_manager = QualityExecutionManager(quality_store)
    engine = EvaluationLoopEngine(learning_store)

    agent = Agent("bot", model=EchoSystemPromptProvider(), system="good answer")
    runner = ContinuousEvalRunner(engine, quality_manager, agent=agent, agent_name="bot")
    runner.jobs["judge-job"] = ContinuousEvalJob(
        name="judge-job",
        kind="judge-eval",
        agent_name="bot",
        workspace_id="acme",
        agent_filter="bot",
        agent=agent,
    )

    with pytest.raises(ValueError, match="requires a judge"):
        await runner.run_job_once("judge-job")

    job = runner.get_job("judge-job")
    assert job is not None
    assert job.last_run_id is not None
    assert job.last_finished_at is not None
    failed = quality_manager.get(job.last_run_id)
    assert failed is not None
    assert failed.status == "failed"
    assert failed.error == "Judge job judge-job requires a judge"
