"""Operator-facing HTTP/CLI surface for the evaluation loop."""
from __future__ import annotations

import asyncio
from datetime import UTC, datetime, timedelta
from itertools import count

import httpx
from click.testing import CliRunner
import pytest

from logos_adk.agent import Agent
from logos_adk.cli import cli
from logos_adk.learning import ContinuousEvalRunner, EvaluationLoopEngine, PromptRegistry, SQLiteLearningStore
from logos_adk.learning.models import FeedbackRecord, OutcomeRecord, RunRecord
from logos_adk.providers.mock import MockProvider
from logos_adk.quality_history import QualityExecutionManager, SQLiteQualityHistoryStore
from logos_adk.types import Response, Usage


class EchoSystemPromptProvider(MockProvider):
    async def generate(self, messages, tools=None, output_schema=None, **kwargs):
        system_prompt = next((message.content for message in messages if message.role == "system"), "")
        return Response(
            content=system_prompt,
            tool_calls=[],
            usage=Usage(input_tokens=20, output_tokens=10),
            raw={},
        )


async def _request(app, method: str, path: str, **kwargs):
    transport = httpx.ASGITransport(app=app)
    async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
        return await client.request(method, path, **kwargs)


def _seed_learning_store(store: SQLiteLearningStore) -> None:
    now = datetime.now(UTC)
    previous_at = now - timedelta(days=10)
    current_at = now - timedelta(days=2)

    def run(
        run_id: str,
        trace_id: str,
        created_at: datetime,
        input_text: str,
        output_text: str,
        *,
        latency_ms: float,
        prompt_variant: str,
    ) -> RunRecord:
        return RunRecord(
            id=run_id,
            trace_id=trace_id,
            request_id=None,
            agent_name="bot",
            input_text=input_text,
            output_text=output_text,
            created_at=created_at.isoformat(),
            session_id="s1",
            workspace_id="acme",
            prompt_version="v1",
            model="gpt-4o-mini",
            usage={"input_tokens": 100, "output_tokens": 50},
            tool_calls=[{"id": "1", "name": "search", "arguments": {}}],
            metadata={"latency_ms": latency_ms, "prompt_variant": prompt_variant},
        )

    store.create_run(run("run-prev", "trace-prev", previous_at, "Prev", "prev", latency_ms=1200, prompt_variant="baseline"))
    store.create_feedback(
        FeedbackRecord(
            id="fb-prev",
            kind="thumbs_down",
            run_id="run-prev",
            trace_id="trace-prev",
            created_at=previous_at.isoformat(),
            workspace_id="acme",
            agent_name="bot",
            prompt_version="v1",
            model="gpt-4o-mini",
            failure_types=["bad_retrieval", "wrong_tool"],
        )
    )
    store.create_feedback(
        FeedbackRecord(
            id="fb-prev-edit",
            kind="edited_answer",
            run_id="run-prev",
            trace_id="trace-prev",
            created_at=previous_at.isoformat(),
            workspace_id="acme",
            agent_name="bot",
            prompt_version="v1",
            model="gpt-4o-mini",
            corrected_output="challenger",
            failure_types=["hallucination"],
        )
    )

    store.create_run(run("run-current", "trace-current", current_at, "Current", "challenger", latency_ms=300, prompt_variant="challenger-v2"))
    store.create_feedback(
        FeedbackRecord(
            id="fb-current",
            kind="thumbs_up",
            run_id="run-current",
            trace_id="trace-current",
            created_at=current_at.isoformat(),
            workspace_id="acme",
            agent_name="bot",
            prompt_version="v1",
            model="gpt-4o-mini",
        )
    )
    store.create_outcome(
        OutcomeRecord(
            id="out-current",
            kind="conversion",
            run_id="run-current",
            trace_id="trace-current",
            created_at=current_at.isoformat(),
            workspace_id="acme",
            agent_name="bot",
            prompt_version="v1",
            model="gpt-4o-mini",
            value=True,
        )
    )


@pytest.mark.asyncio
async def test_http_surface_exposes_evaluation_loop_state(tmp_path, monkeypatch):
    from logos_adk.serve import create_app

    continuous_eval_db = tmp_path / "continuous-eval.db"
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    prompt_registry = PromptRegistry(str(tmp_path / "prompts.db"))
    prompt_registry.ensure_prompt("bot")
    base = prompt_registry.create_version("bot", content="baseline")
    prompt_registry.promote("bot", version=base.version, workspace_id="acme")
    _seed_learning_store(learning_store)
    agent = Agent("bot", model=EchoSystemPromptProvider(), system="baseline")
    monkeypatch.setenv("LOGOS_ADK_CONTINUOUS_EVAL_DB_PATH", str(continuous_eval_db))
    app = create_app([agent], learning_store=learning_store, prompt_registry=prompt_registry)

    scorecard = await _request(
        app,
        "GET",
        "/learning/evaluation/scorecard?workspace_id=acme&agent_name=bot&window_days=7",
    )
    dataset = await _request(
        app,
        "GET",
        "/learning/evaluation/regression-dataset?workspace_id=acme&agent_name=bot&limit=20",
    )
    disagreements = await _request(
        app,
        "POST",
        "/learning/evaluation/judge-disagreements",
        json={
            "workspace_id": "acme",
            "agent_name": "bot",
            "limit": 20,
            "threshold": 0.35,
            "pass_threshold": 0.7,
            "mock_response": '{"score": 0.95, "rationale": "looks excellent"}',
        },
    )
    champion = await _request(
        app,
        "POST",
        "/learning/evaluation/champion-challenger",
        json={
            "workspace_id": "acme",
            "agent_name": "bot",
            "champion_prompt": "baseline",
            "challenger_prompt": "challenger",
            "champion_filter": {"prompt_variant": "baseline"},
            "challenger_filter": {"prompt_variant": "challenger-v2"},
            "limit": 20,
            "reward_scorer_mode": "learned",
            "reward_training_limit": 50,
        },
    )
    create_job = await _request(
        app,
        "POST",
        "/learning/evaluation/jobs",
        json={
            "name": "prod-regression",
            "kind": "regression",
            "workspace_id": "acme",
            "agent_name": "bot",
            "interval_seconds": 60.0,
            "limit": 20,
            "metadata": {
                "self_learning_policy": {
                    "prompt_name": "bot",
                    "workspace_id": "acme",
                    "environment": "prod",
                    "min_pass_rate": 2.0,
                    "min_total_cases": 1,
                    "auto_rollback_on_regression": False,
                }
            },
        },
    )
    jobs = await _request(app, "GET", "/learning/evaluation/jobs")
    run_job = await _request(app, "POST", "/learning/evaluation/jobs/prod-regression/run")
    trend_deltas = await _request(app, "GET", "/learning/evaluation/trend-deltas?window_days=7")
    incidents = await _request(app, "GET", "/learning/evaluation/rollback-incidents?limit=20")
    incident_snapshot = await _request(app, "GET", "/learning/evaluation/incident-snapshot?limit=10")
    incident_id = incidents.json()["items"][0]["id"]
    ack_incident = await _request(
        app,
        "POST",
        f"/learning/evaluation/rollback-incidents/{incident_id}/ack",
        json={"actor": "ops", "note": "triaged"},
    )
    resolve_incident = await _request(
        app,
        "POST",
        f"/learning/evaluation/rollback-incidents/{incident_id}/resolve",
        json={"actor": "ops", "note": "handled"},
    )
    restored_app = create_app([agent], learning_store=learning_store, prompt_registry=prompt_registry)
    restored_jobs = await _request(restored_app, "GET", "/learning/evaluation/restored-jobs")
    delete_job = await _request(app, "DELETE", "/learning/evaluation/jobs/prod-regression")

    assert scorecard.status_code == 200
    assert dataset.status_code == 200
    assert disagreements.status_code == 200
    assert champion.status_code == 200
    assert create_job.status_code == 200
    assert jobs.status_code == 200
    assert run_job.status_code == 200
    assert trend_deltas.status_code == 200
    assert incidents.status_code == 200
    assert incident_snapshot.status_code == 200
    assert ack_incident.status_code == 200
    assert resolve_incident.status_code == 200
    assert delete_job.status_code == 200
    assert restored_jobs.status_code == 200

    scorecard_payload = scorecard.json()["scorecard"]
    assert scorecard_payload["quality"]["delta"] > 0
    assert scorecard_payload["retrieval_relevance"]["delta"] > 0

    dataset_payload = dataset.json()["dataset"]
    assert len(dataset_payload["examples"]) >= 1
    assert dataset_payload["examples"][0]["expected"] == "challenger"

    disagreements_payload = disagreements.json()
    assert disagreements_payload["summary"]["judge_disagreements"] >= 1
    assert disagreements_payload["items"][0]["judge_passed"] is True
    assert disagreements_payload["items"][0]["human_passed"] is False

    champion_payload = champion.json()["report"]
    assert champion_payload["recommendation"] == "challenger"
    assert champion_payload["reward_scorer_mode"] == "learned"
    assert champion_payload["challenger"]["offline"]["pass_rate"] >= champion_payload["champion"]["offline"]["pass_rate"]

    jobs_payload = jobs.json()["items"]
    assert any(item["name"] == "prod-regression" for item in jobs_payload)
    assert run_job.json()["run"]["status"] == "completed"
    trend_payload = trend_deltas.json()["report"]
    assert trend_payload["incidents"]["current"] >= 1.0
    incident_payload = incidents.json()["items"]
    assert any(item["kind"] == "regression_threshold_breach" for item in incident_payload)
    incident_snapshot_payload = incident_snapshot.json()["snapshot"]
    assert incident_snapshot_payload["summary"]["total_incidents"] >= 1
    assert incident_snapshot_payload["summary"]["by_kind"]["regression_threshold_breach"] >= 1
    assert ack_incident.json()["incident"]["status"] == "acknowledged"
    assert resolve_incident.json()["incident"]["status"] == "resolved"
    assert delete_job.json()["job"]["name"] == "prod-regression"
    assert any(item["name"] == "prod-regression" and item["restored"] is True for item in restored_jobs.json()["items"])


def test_cli_surface_exposes_evaluation_loop_state(tmp_path, monkeypatch):
    learning_store = SQLiteLearningStore(str(tmp_path / "learning.db"))
    _seed_learning_store(learning_store)
    config = tmp_path / "agent.yaml"
    config.write_text("name: bot\nmodel: mock\nsystem_prompt: baseline\n")
    continuous_eval_db = tmp_path / "continuous-eval.db"
    prompt_registry = PromptRegistry(str(tmp_path / "prompts.db"))
    prompt_registry.ensure_prompt("bot")
    base = prompt_registry.create_version("bot", content="baseline")
    prompt_registry.promote("bot", version=base.version, workspace_id="acme")

    seq = count()
    monkeypatch.setattr(
        "logos_adk.cli.Agent.from_config",
        lambda path: Agent(f"bot-cli-{next(seq)}", model=EchoSystemPromptProvider(), system="baseline"),
    )

    runner = CliRunner()
    scorecard = runner.invoke(
        cli,
        [
            "evaluation",
            "scorecard",
            "--db-path",
            str(tmp_path / "learning.db"),
            "--workspace-id",
            "acme",
            "--agent-name",
            "bot",
        ],
    )
    regression = runner.invoke(
        cli,
        [
            "evaluation",
            "regression-dataset",
            "--db-path",
            str(tmp_path / "learning.db"),
            "--workspace-id",
            "acme",
            "--agent-name",
            "bot",
        ],
    )
    disagreements = runner.invoke(
        cli,
        [
            "evaluation",
            "judge-disagreements",
            str(config),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--workspace-id",
            "acme",
            "--agent-name",
            "bot",
            "--mock-response",
            '{"score": 0.95, "rationale": "excellent"}',
        ],
    )
    champion = runner.invoke(
        cli,
        [
            "evaluation",
            "champion-challenger",
            str(config),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--workspace-id",
            "acme",
            "--agent-name",
            "bot",
            "--champion-prompt",
            "baseline",
            "--challenger-prompt",
            "challenger",
            "--champion-filter-json",
            '{"prompt_variant":"baseline"}',
            "--challenger-filter-json",
            '{"prompt_variant":"challenger-v2"}',
        ],
    )
    run_job = runner.invoke(
        cli,
        [
            "evaluation",
            "run-regression-job",
            str(config),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
            "--workspace-id",
            "acme",
            "--agent-name",
            "bot",
            "--name",
            "reg-job",
        ],
    )
    quality_manager = QualityExecutionManager(SQLiteQualityHistoryStore(str(tmp_path / "quality.db")))
    eval_runner = ContinuousEvalRunner(
        EvaluationLoopEngine(learning_store),
        quality_manager,
        path=str(continuous_eval_db),
        agent=Agent("bot", model=EchoSystemPromptProvider(), system="baseline"),
        agent_name="bot",
        prompt_registry=prompt_registry,
    )
    eval_runner.register_regression_job(
        "restore-job",
        agent=Agent("bot", model=EchoSystemPromptProvider(), system="baseline"),
        workspace_id="acme",
        agent_name="bot",
        interval_seconds=60.0,
        limit=20,
        metadata={
            "self_learning_policy": {
                "prompt_name": "bot",
                "workspace_id": "acme",
                "environment": "prod",
                "min_pass_rate": 2.0,
                "min_total_cases": 1,
                "auto_rollback_on_regression": False,
            }
        },
    )
    asyncio_result = eval_runner.run_job_once("restore-job")
    asyncio.run(asyncio_result)
    restored_jobs = runner.invoke(
        cli,
        [
            "evaluation",
            "restored-jobs",
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
        ],
    )
    trend_deltas = runner.invoke(
        cli,
        [
            "evaluation",
            "trend-deltas",
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
            "--workspace-id",
            "acme",
        ],
    )
    incidents = runner.invoke(
        cli,
        [
            "evaluation",
            "rollback-incidents",
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
            "--workspace-id",
            "acme",
        ],
    )
    incident_snapshot = runner.invoke(
        cli,
        [
            "evaluation",
            "incident-snapshot",
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
            "--workspace-id",
            "acme",
        ],
    )
    runner_incidents = _build_continuous_eval_runner_for_test(
        learning_store=learning_store,
        quality_db_path=str(tmp_path / "quality.db"),
        continuous_eval_db_path=str(continuous_eval_db),
        prompt_registry=prompt_registry,
    )
    incident_rows = runner_incidents.list_incidents(limit=10, workspace_id="acme")
    incident_id = incident_rows[0]["id"]
    ack_incident = runner.invoke(
        cli,
        [
            "evaluation",
            "ack-incident",
            incident_id,
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
            "--actor",
            "ops",
            "--note",
            "triaged",
        ],
    )
    resolve_incident = runner.invoke(
        cli,
        [
            "evaluation",
            "resolve-incident",
            incident_id,
            "--continuous-eval-db-path",
            str(continuous_eval_db),
            "--db-path",
            str(tmp_path / "learning.db"),
            "--quality-db-path",
            str(tmp_path / "quality.db"),
            "--prompt-registry-path",
            str(tmp_path / "prompts.db"),
            "--actor",
            "ops",
            "--note",
            "handled",
        ],
    )

    assert scorecard.exit_code == 0
    assert "quality_delta=" in scorecard.output
    assert regression.exit_code == 0
    assert "examples=" in regression.output
    assert disagreements.exit_code == 0
    assert "disagreements=" in disagreements.output
    assert champion.exit_code == 0
    assert "Recommendation: challenger" in champion.output
    assert run_job.exit_code == 0
    assert "reg-job: status=completed" in run_job.output
    assert restored_jobs.exit_code == 0
    assert "restore-job: kind=regression restored=True" in restored_jobs.output
    assert trend_deltas.exit_code == 0
    assert "incidents_delta=" in trend_deltas.output
    assert incidents.exit_code == 0
    assert "regression_threshold_breach" in incidents.output
    assert incident_snapshot.exit_code == 0
    assert "total_incidents=" in incident_snapshot.output
    assert ack_incident.exit_code == 0
    assert "status=acknowledged" in ack_incident.output
    assert resolve_incident.exit_code == 0
    assert "status=resolved" in resolve_incident.output


def _build_continuous_eval_runner_for_test(*, learning_store, quality_db_path: str, continuous_eval_db_path: str, prompt_registry: PromptRegistry):
    return ContinuousEvalRunner(
        EvaluationLoopEngine(learning_store),
        QualityExecutionManager(SQLiteQualityHistoryStore(quality_db_path)),
        path=continuous_eval_db_path,
        prompt_registry=prompt_registry,
    )
