# ruff: noqa: T201
"""
Evaluate the `search_functions` meta-tool against a curated set of queries.

The script mirrors the legacy FastAPI search tester (`test.py`) but uses the
CodeAct0 `search_functions` tool directly. It loads expected matches from
`src/evals/datasets/search.json`, executes searches, computes simple accuracy
metrics, and prints a concise report.
"""

from __future__ import annotations

import asyncio
import json
import re
import statistics
import time
from dataclasses import dataclass
from pathlib import Path

from universal_mcp.agentr.registry import AgentrRegistry
from universal_mcp.agents.codeact0.tools import create_meta_tools


# ----------------------------- Data structures ----------------------------- #
@dataclass
class TestResult:
    query: str
    expected_exact: list[str]
    expected_others: list[str]
    returned_tools: list[str]
    all_exact_in_top5: bool
    all_exact_in_top10: bool
    all_exact_in_top20: bool
    others_percentage_in_top10: float
    exact_percentage_in_top10: float
    others_percentage_in_top20: float
    exact_percentage_in_top20: float
    response_time: float
    error: str | None = None


# ------------------------------ Core helpers ------------------------------- #
TOOL_ID_PATTERN = re.compile(r"-\s+([a-zA-Z0-9_-]+__[^:\s]+)")


def parse_tool_ids(output: str) -> list[str]:
    """
    Extract tool ids from the `search_functions` formatted string.

    The tool output looks like:
        Tools from gmail (status: connected by user):
         - google_mail__send_email: ...
    """

    return TOOL_ID_PATTERN.findall(output)


def calculate_metrics(
    expected_exact: list[str], expected_others: list[str], returned: list[str]
) -> tuple[bool, bool, bool, float, float, float, float]:
    if not returned:
        return False, False, False, 0.0, 0.0, 0.0, 0.0

    top5 = returned[:5]
    top10 = returned[:10]
    top20 = returned[:20]

    all_exact_in_top5 = all(tool in top5 for tool in expected_exact) if expected_exact else True
    all_exact_in_top10 = all(tool in top10 for tool in expected_exact) if expected_exact else True
    all_exact_in_top20 = all(tool in top20 for tool in expected_exact) if expected_exact else True

    others_in_top10 = sum(1 for tool in expected_others if tool in top10)
    exact_in_top10 = sum(1 for tool in expected_exact if tool in top10)
    others_in_top20 = sum(1 for tool in expected_others if tool in top20)
    exact_in_top20 = sum(1 for tool in expected_exact if tool in top20)

    others_pct_10 = (others_in_top10 / len(expected_others) * 100) if expected_others else 0.0
    exact_pct_10 = (exact_in_top10 / len(expected_exact) * 100) if expected_exact else 0.0
    others_pct_20 = (others_in_top20 / len(expected_others) * 100) if expected_others else 0.0
    exact_pct_20 = (exact_in_top20 / len(expected_exact) * 100) if expected_exact else 0.0

    return (
        all_exact_in_top5,
        all_exact_in_top10,
        all_exact_in_top20,
        others_pct_10,
        exact_pct_10,
        others_pct_20,
        exact_pct_20,
    )


async def run_single_test(search_fn, query: str, expected_exact: list[str], expected_others: list[str]) -> TestResult:
    start = time.time()
    try:
        raw_output = await search_fn.ainvoke({"requests": [{"query": query}]})
        response_time = time.time() - start
        returned_tools = parse_tool_ids(raw_output)
        metrics = calculate_metrics(expected_exact, expected_others, returned_tools)
        return TestResult(
            query=query,
            expected_exact=expected_exact,
            expected_others=expected_others,
            returned_tools=returned_tools,
            all_exact_in_top5=metrics[0],
            all_exact_in_top10=metrics[1],
            all_exact_in_top20=metrics[2],
            others_percentage_in_top10=metrics[3],
            exact_percentage_in_top10=metrics[4],
            others_percentage_in_top20=metrics[5],
            exact_percentage_in_top20=metrics[6],
            response_time=response_time,
        )
    except Exception as exc:  # noqa: BLE001
        response_time = time.time() - start
        return TestResult(
            query=query,
            expected_exact=expected_exact,
            expected_others=expected_others,
            returned_tools=[],
            all_exact_in_top5=False,
            all_exact_in_top10=False,
            all_exact_in_top20=False,
            others_percentage_in_top10=0.0,
            exact_percentage_in_top10=0.0,
            others_percentage_in_top20=0.0,
            exact_percentage_in_top20=0.0,
            response_time=response_time,
            error=str(exc),
        )


# ------------------------------- Reporting -------------------------------- #
def print_test_result(result: TestResult) -> None:
    print("\n" + "=" * 80)
    print(f"Query: {result.query}")
    print("=" * 80)

    if result.error:
        print(f"❌ ERROR: {result.error}")
        return

    print(f"Expected Exact Matches: {result.expected_exact}")
    print(f"Expected Others: {result.expected_others}")
    print(f"Response Time: {result.response_time:.3f}s")
    print(f"All Exact in Top 5: {'✅' if result.all_exact_in_top5 else '❌'}")
    print(f"All Exact in Top 10: {'✅' if result.all_exact_in_top10 else '❌'}")
    print(f"All Exact in Top 20: {'✅' if result.all_exact_in_top20 else '❌'}")
    print(f"Exact in Top 10: {result.exact_percentage_in_top10:.1f}%")
    print(f"Exact in Top 20: {result.exact_percentage_in_top20:.1f}%")
    print(f"Others in Top 10: {result.others_percentage_in_top10:.1f}%")
    print(f"Others in Top 20: {result.others_percentage_in_top20:.1f}%")

    print(f"\nReturned Tools ({len(result.returned_tools)}):")
    for idx, tool_id in enumerate(result.returned_tools, start=1):
        if tool_id in result.expected_exact:
            badge = "✅"
        elif tool_id in result.expected_others:
            badge = "🔵"
        else:
            badge = "⚪"
        print(f"  {idx:2d}. {badge} {tool_id}")


def print_summary(results: list[TestResult]) -> None:
    print("\n" + "=" * 80)
    print("TEST SUMMARY")
    print("=" * 80)

    successes = [r for r in results if not r.error]
    failures = [r for r in results if r.error]

    print(f"Total Tests: {len(results)}")
    print(f"Successful: {len(successes)}")
    print(f"Failed: {len(failures)}")

    if successes:
        response_times = [r.response_time for r in successes]
        exact_top5_count = sum(1 for r in successes if r.all_exact_in_top5)
        exact_top10_count = sum(1 for r in successes if r.all_exact_in_top10)
        exact_top20_count = sum(1 for r in successes if r.all_exact_in_top20)
        others_pcts_10 = [r.others_percentage_in_top10 for r in successes]
        exact_pcts_10 = [r.exact_percentage_in_top10 for r in successes]
        others_pcts_20 = [r.others_percentage_in_top20 for r in successes]
        exact_pcts_20 = [r.exact_percentage_in_top20 for r in successes]

        print("\nExact Match Statistics:")
        print(
            f"  All Exact in Top 5: {exact_top5_count}/{len(successes)} ({exact_top5_count / len(successes) * 100:.1f}%)"
        )
        print(
            f"  All Exact in Top 10: {exact_top10_count}/{len(successes)} ({exact_top10_count / len(successes) * 100:.1f}%)"
        )
        print(
            f"  All Exact in Top 20: {exact_top20_count}/{len(successes)} ({exact_top20_count / len(successes) * 100:.1f}%)"
        )

        print("\nExact Percentage Statistics (Top 10):")
        print(f"  Average Exact in Top 10: {statistics.mean(exact_pcts_10):.1f}%")
        print(f"  Median Exact in Top 10:  {statistics.median(exact_pcts_10):.1f}%")
        print(f"  Min Exact in Top 10:     {min(exact_pcts_10):.1f}%")
        print(f"  Max Exact in Top 10:     {max(exact_pcts_10):.1f}%")

        print("\nExact Percentage Statistics (Top 20):")
        print(f"  Average Exact in Top 20: {statistics.mean(exact_pcts_20):.1f}%")
        print(f"  Median Exact in Top 20:  {statistics.median(exact_pcts_20):.1f}%")
        print(f"  Min Exact in Top 20:     {min(exact_pcts_20):.1f}%")
        print(f"  Max Exact in Top 20:     {max(exact_pcts_20):.1f}%")

        print("\nOthers Match Statistics (Top 10):")
        print(f"  Average Others in Top 10: {statistics.mean(others_pcts_10):.1f}%")
        print(f"  Median Others in Top 10:  {statistics.median(others_pcts_10):.1f}%")
        print(f"  Min Others in Top 10:     {min(others_pcts_10):.1f}%")
        print(f"  Max Others in Top 10:     {max(others_pcts_10):.1f}%")

        print("\nOthers Match Statistics (Top 20):")
        print(f"  Average Others in Top 20: {statistics.mean(others_pcts_20):.1f}%")
        print(f"  Median Others in Top 20:  {statistics.median(others_pcts_20):.1f}%")
        print(f"  Min Others in Top 20:     {min(others_pcts_20):.1f}%")
        print(f"  Max Others in Top 20:     {max(others_pcts_20):.1f}%")

        print("\nResponse Time Statistics:")
        print(f"  Average Time:  {statistics.mean(response_times):.3f}s")
        print(f"  Median Time:   {statistics.median(response_times):.3f}s")
        print(f"  Min Time:      {min(response_times):.3f}s")
        print(f"  Max Time:      {max(response_times):.3f}s")

    if failures:
        print("\nFailed Tests:")
        for res in failures:
            print(f"  ❌ {res.query}: {res.error}")

    if successes:
        sorted_results = sorted(
            successes,
            key=lambda r: (
                r.all_exact_in_top5,
                r.all_exact_in_top10,
                r.all_exact_in_top20,
                r.exact_percentage_in_top10,
                r.others_percentage_in_top10,
            ),
            reverse=True,
        )
        print("\nTop 3 Performers (by exact matches):")
        for idx, res in enumerate(sorted_results[:3], start=1):
            print(f"  {idx}. {res.query}")
            print(
                f"     Top5: {'✅' if res.all_exact_in_top5 else '❌'}, Top10: {'✅' if res.all_exact_in_top10 else '❌'}, Top20: {'✅' if res.all_exact_in_top20 else '❌'}, Exact% (10): {res.exact_percentage_in_top10:.1f}%, Others (10): {res.others_percentage_in_top10:.1f}%"
            )

        print("\nBottom 3 Performers (by exact matches):")
        for idx, res in enumerate(sorted_results[-3:], start=1):
            print(f"  {idx}. {res.query}")
            print(
                f"     Top5: {'✅' if res.all_exact_in_top5 else '❌'}, Top10: {'✅' if res.all_exact_in_top10 else '❌'}, Top20: {'✅' if res.all_exact_in_top20 else '❌'}, Exact% (10): {res.exact_percentage_in_top10:.1f}%, Others (10): {res.others_percentage_in_top10:.1f}%"
            )


# ------------------------------- Main runner ------------------------------- #
async def load_test_data(file_path: Path) -> dict[str, dict[str, list[str]]]:
    if not file_path.exists():
        raise FileNotFoundError(f"Test data file not found: {file_path}")
    with file_path.open("r", encoding="utf-8") as f:
        return json.load(f)


async def main() -> int:
    print("CodeAct0 search_functions Test Suite")
    print("=" * 50)

    data_path = Path(__file__).parent / "datasets" / "search.json"
    test_data = await load_test_data(data_path)
    print(f"Loaded {len(test_data)} test queries from {data_path}")

    registry = AgentrRegistry()
    meta_tools = create_meta_tools(registry)
    search_fn = meta_tools["search_functions"]

    results: list[TestResult] = []
    print("\nRunning tests...")

    for idx, (query, expected) in enumerate(test_data.items(), start=1):
        print(f"\n[{idx}/{len(test_data)}] Testing: {query}")
        expected_exact = expected.get("exact", [])
        expected_others = expected.get("others", [])
        result = await run_single_test(search_fn, query, expected_exact, expected_others)
        results.append(result)
        print_test_result(result)
        await asyncio.sleep(0.05)

    print_summary(results)
    return 0


if __name__ == "__main__":
    raise SystemExit(asyncio.run(main()))
