"""Validation runner — executes test suites through the real runtime.

Boots a lightweight in-process AlbusOS stack (no HTTP server), runs each
test case via ``runtime.turn()``, and checks assertions against the result.
"""

from __future__ import annotations

import logging
import time
from pathlib import Path
from typing import Any

import yaml

from core.validation.models import (
    AssertionFailure,
    CaseResult,
    ValidationCase,
    ValidationResult,
    ValidationSuite,
)

logger = logging.getLogger(__name__)


class ValidationRunner:
    """Runs validation suites through the real AlbusOS engine."""

    def __init__(self, *, verbose: bool = False, stop_on_fail: bool = False) -> None:
        self.verbose = verbose
        self.stop_on_fail = stop_on_fail

    # -----------------------------------------------------------------
    # Public API
    # -----------------------------------------------------------------

    async def run_suite(self, suite_path: str | Path) -> ValidationResult:
        """Load and execute a validation suite YAML.

        Args:
            suite_path: Path to the validation suite YAML file.

        Returns
        -------
            ValidationResult with per-case results and aggregate stats.
        """
        suite_path = Path(suite_path).resolve()
        if not suite_path.exists():
            return ValidationResult(
                suite_id="unknown",
                passed=False,
                setup_error=f"Suite file not found: {suite_path}",
            )

        # Load suite
        with open(suite_path) as f:
            raw = yaml.safe_load(f) or {}

        suite = ValidationSuite.model_validate(raw)
        self._log(f"Suite: {suite.name or suite.id} ({len(suite.cases)} cases)")

        # Boot the engine
        engine, agent_repo = await self._boot_engine()

        # Run setup
        setup_error = await self._run_setup(suite, suite_path.parent)
        if setup_error:
            return ValidationResult(
                suite_id=suite.id,
                suite_name=suite.name,
                passed=False,
                setup_error=setup_error,
            )

        # Execute cases
        start = time.monotonic()
        results: list[CaseResult] = []
        for case in suite.cases:
            agent_id = case.agent or suite.agent
            pathway = case.pathway or suite.pathway

            case_result = await self._run_case(
                case=case,
                agent_id=agent_id,
                pathway_override=pathway,
                engine=engine,
            )
            results.append(case_result)

            status = "PASS" if case_result.passed else "FAIL"
            self._log(f"  [{status}] {case.id} ({case_result.duration_ms}ms)")

            if not case_result.passed:
                for f in case_result.failures:
                    self._log(f"    - {f.check}: expected {f.expected}, got {f.actual}")
                if case_result.error:
                    self._log(f"    - error: {case_result.error}")

            if self.stop_on_fail and not case_result.passed:
                break

        total_ms = int((time.monotonic() - start) * 1000)

        # Teardown
        await self._run_teardown(suite)

        # Aggregate
        passed_count = sum(1 for r in results if r.passed)
        failed_count = sum(1 for r in results if not r.passed)
        all_passed = failed_count == 0

        return ValidationResult(
            suite_id=suite.id,
            suite_name=suite.name,
            passed=all_passed,
            cases_run=len(results),
            cases_passed=passed_count,
            cases_failed=failed_count,
            total_duration_ms=total_ms,
            results=results,
        )

    # -----------------------------------------------------------------
    # Engine boot
    # -----------------------------------------------------------------

    async def _boot_engine(self) -> tuple[Any, Any]:
        """Boot a lightweight AlbusOS runtime (no HTTP server).

        Returns
        -------
            (Runtime, AgentRepository)
        """
        from core.pathways.context import Context
        from core.pathways.vm import PathwayVM
        from core.tools import get_tool_registry
        from server.config import DeploymentConfig
        from server.loaders import (
            load_agents_from_directories,
            load_builtin_skills,
        )
        from server.runtime import Runtime
        from stdlib.bootstrap import load_stdlib

        # 1. Bootstrap stdlib (tools + skills)
        load_stdlib()
        registry = get_tool_registry()

        # 2. Load deployment config
        config = DeploymentConfig.load()

        # 3. Build Context + VM
        ctx = Context(tools={td.name: td.handler for td in registry.list() if td.handler})
        vm = PathwayVM(ctx)

        # 4. Create runtime (engine + repos)
        runtime = await Runtime.create(pathway_vm=vm)

        # 5. Apply model routing
        if config.models and config.models.routing:
            from server.loaders import apply_model_config  # noqa: PLC0415 (circular)

            apply_model_config(
                config.models.default_profile,
                config.models.routing,
            )

        # 6. Register agents from config
        if config.agent_specs:
            await runtime.register_agents(config.agent_specs)

        # 6b. Agent directories
        if config.agent_directories:
            try:
                await load_agents_from_directories(
                    config.agent_directories,
                    runtime.agent_repo,
                )
            except Exception as e:
                logger.warning("Failed to load agents from directories: %s", e)

        # 7. Skills
        if config.skills and config.skills.directories:
            try:
                await runtime.load_skills(
                    config.skills.directories,
                    auto_activate=config.skills.auto_activate,
                )
            except Exception as e:
                logger.warning("Failed to load skills: %s", e)

        # 8. Pathways
        if config.pathways_config and config.pathways_config.directories:
            try:
                await runtime.load_pathways(
                    config.pathways_config.directories,
                    auto_deploy=config.pathways_config.auto_deploy,
                )
            except Exception as e:
                logger.warning("Failed to load pathways: %s", e)

        # 9. Builtin skills
        try:
            await load_builtin_skills()
        except Exception:
            logger.debug("Failed to load builtin skills", exc_info=True)

        self._log("Engine booted")
        return runtime, runtime.agent_repo

    # -----------------------------------------------------------------
    # Setup / Teardown
    # -----------------------------------------------------------------

    async def _run_setup(self, suite: ValidationSuite, suite_dir: Path) -> str | None:
        """Run setup steps (e.g. document ingestion).

        Calls ``knowledge.ingest`` through the tool registry if available.
        Requires a knowledge tool to be registered (not included in stdlib).

        Returns
        -------
            Error message if setup failed, None if OK.
        """
        if not suite.setup.ingest:
            return None

        self._log("Running setup...")

        try:
            from core.pathways.models.tool import ToolArgs
            from core.tools import get_tool_registry

            registry = get_tool_registry()

            for ingest_spec in suite.setup.ingest:
                # Resolve relative paths against suite directory
                ingest_path = Path(ingest_spec.path)
                if not ingest_path.is_absolute():
                    ingest_path = suite_dir / ingest_path

                self._log(f"  Ingesting: {ingest_path} -> {ingest_spec.collection}")

                args = ToolArgs(
                    path=str(ingest_path),
                    collection=ingest_spec.collection,
                    chunk_size=ingest_spec.chunk_size,
                    chunk_overlap=ingest_spec.chunk_overlap,
                )
                result = await registry.call("knowledge.ingest", args)

                if not result.success:
                    return f"Ingest failed for {ingest_spec.path}: {result.error}"

                data = result.data if isinstance(result.data, dict) else {}
                self._log(
                    f"    Ingested {data.get('files_ingested', 0)} files, "
                    f"{data.get('chunks_created', 0)} chunks"
                )

        except Exception as e:
            return f"Setup failed: {e}"

        return None

    async def _run_teardown(self, suite: ValidationSuite) -> None:
        """Run teardown steps (e.g. cleanup collections)."""
        if not suite.teardown.cleanup_collections:
            return

        self._log("Running teardown...")

        import shutil

        from core.validation.runner import _get_store_root

        store_root = _get_store_root()
        for collection in suite.teardown.cleanup_collections:
            col_path = store_root / collection
            if col_path.exists():
                shutil.rmtree(col_path, ignore_errors=True)
                self._log(f"  Cleaned up collection: {collection}")

    # -----------------------------------------------------------------
    # Case execution
    # -----------------------------------------------------------------

    async def _run_case(
        self,
        *,
        case: ValidationCase,
        agent_id: str,
        pathway_override: str | None,
        engine: Any,
    ) -> CaseResult:
        """Execute a single validation case through the runtime."""
        start = time.monotonic()

        try:
            result = await engine.turn(
                agent_id=agent_id,
                message=case.input,
                thread_id=f"validate-{case.id}",
                pathway_override=pathway_override,
            )
        except Exception as e:
            elapsed = int((time.monotonic() - start) * 1000)
            return CaseResult(
                case_id=case.id,
                passed=False,
                duration_ms=elapsed,
                error=str(e),
            )

        elapsed = int((time.monotonic() - start) * 1000)

        # Extract data from ExecutionResult
        response = result.response or ""
        tools_called = [tc.tool_name for tc in (result.tool_calls or [])]
        tool_call_count = len(result.tool_calls or [])

        # Check assertions
        failures = _check_assertions(case, response, tools_called, tool_call_count, result)

        return CaseResult(
            case_id=case.id,
            passed=len(failures) == 0,
            response=response,
            tools_called=tools_called,
            tool_call_count=tool_call_count,
            steps_taken=result.steps_taken,
            duration_ms=elapsed,
            tokens_used=result.tokens_used,
            cost_usd=result.cost_usd,
            failures=failures,
        )

    # -----------------------------------------------------------------
    # Helpers
    # -----------------------------------------------------------------

    def _log(self, msg: str) -> None:
        """Log if verbose mode is on."""
        if self.verbose:
            logger.info(msg)


# ---------------------------------------------------------------------------
# Assertion checking
# ---------------------------------------------------------------------------


def _check_assertions(
    case: ValidationCase,
    response: str,
    tools_called: list[str],
    tool_call_count: int,
    result: Any,
) -> list[AssertionFailure]:
    """Check all assertions for a case. Returns list of failures."""
    failures: list[AssertionFailure] = []
    expect = case.expect
    response_lower = response.lower()

    # -- success --
    if expect.success and not result.success:
        failures.append(AssertionFailure(
            check="success",
            expected="True",
            actual="False",
        ))
    elif not expect.success and result.success:
        failures.append(AssertionFailure(
            check="success",
            expected="False (expected failure)",
            actual="True (succeeded)",
        ))

    # -- tools_used --
    # Normalize tool names: LLMs convert dots to underscores in tool calls
    # e.g. "knowledge.search" → "knowledge_search"
    def _normalize_tool_name(name: str) -> str:
        return name.replace(".", "_")

    normalized_calls = [_normalize_tool_name(tc) for tc in tools_called]

    for tool in expect.tools_used:
        norm_expected = _normalize_tool_name(tool)
        if not any(norm_expected in nc for nc in normalized_calls):
            failures.append(AssertionFailure(
                check="tools_used",
                expected=f"'{tool}' in tool calls",
                actual=f"tools called: {tools_called}",
            ))

    # -- min_tool_calls --
    if expect.min_tool_calls > 0 and tool_call_count < expect.min_tool_calls:
        failures.append(AssertionFailure(
            check="min_tool_calls",
            expected=f">= {expect.min_tool_calls}",
            actual=str(tool_call_count),
        ))

    # -- response_contains --
    for substring in expect.response_contains:
        if substring.lower() not in response_lower:
            failures.append(AssertionFailure(
                check="response_contains",
                expected=f"contains '{substring}'",
                actual=f"response: {response[:200]}...",
            ))

    # -- response_not_contains --
    for substring in expect.response_not_contains:
        if substring.lower() in response_lower:
            failures.append(AssertionFailure(
                check="response_not_contains",
                expected=f"does NOT contain '{substring}'",
                actual=f"response: {response[:200]}...",
            ))

    # -- response_contains_any --
    if expect.response_contains_any:
        found = any(s.lower() in response_lower for s in expect.response_contains_any)
        if not found:
            failures.append(AssertionFailure(
                check="response_contains_any",
                expected=f"contains any of {expect.response_contains_any}",
                actual=f"response: {response[:200]}...",
            ))

    # -- max_steps --
    if expect.max_steps is not None and result.steps_taken > expect.max_steps:
        failures.append(AssertionFailure(
            check="max_steps",
            expected=f"<= {expect.max_steps}",
            actual=str(result.steps_taken),
        ))

    return failures


def _get_store_root() -> Path:
    """Get the knowledge store root path."""
    import os
    return Path(os.getenv("ALBUS_STORE_DIR", ".albus_store")) / "knowledge"
