"""搜索质量评测 — 端到端验证 unified_recall

需要真实数据库（~/.claude-index1/），不能 mock。
用 pytest -m eval 运行，普通 pytest 跳过。

输出: P@5, MRR, Hit@10 — 按查询类型分组汇总。
"""

from __future__ import annotations

import asyncio
import json
import os
from pathlib import Path

import pytest

from index1.config import Config, INDEX1_HOME, load_config
from index1.db import Database
from index1.embed import create_embedder

from .eval_cases import ALL_CASES, EvalCase

# ── 标记: pytest -m eval 才运行 ──

pytestmark = pytest.mark.eval

# 跳过条件：没有真实数据库
_CORPUS_DB = INDEX1_HOME / "knowledge.db"
_COG_DB = INDEX1_HOME / "cognitive.db"
_SKIP_REASON = "需要真实数据库 ~/.claude-index1/"

if not _CORPUS_DB.exists() or not _COG_DB.exists():
    pytestmark = [pytestmark, pytest.mark.skip(reason=_SKIP_REASON)]


# ── Fixtures ──


@pytest.fixture(scope="module")
def config():
    return load_config()


@pytest.fixture(scope="module")
def corpus_db(config):
    db = Database(config.db_path, embedding_dim=config.effective_embedding_dim)
    db.connect()
    yield db
    db.close()


@pytest.fixture(scope="module")
def cog_db(config):
    from index1.cognitive.db_cog import CogDatabase
    db = CogDatabase(config.cognitive_db_path, embedding_dim=config.effective_embedding_dim)
    db.connect()
    yield db
    db.close()


@pytest.fixture(scope="module")
def embedder(config):
    emb = create_embedder(config)
    return emb


# ── 评测指标计算 ──


def _check_code_hit(result_source: str | None, expected: list[str]) -> bool:
    """检查 corpus 结果是否命中 expected 文件路径子串"""
    if not result_source or not expected:
        return False
    source_lower = result_source.lower()
    return any(exp.lower() in source_lower for exp in expected)


def _check_fact_hit(assertion: str, expected: list[str]) -> bool:
    """检查 cognition 结果是否命中 expected 关键词"""
    if not assertion or not expected:
        return False
    assertion_lower = assertion.lower()
    # 至少命中一半关键词算 hit
    hits = sum(1 for kw in expected if kw.lower() in assertion_lower)
    return hits >= max(1, len(expected) // 2)


def compute_metrics(
    case: EvalCase,
    code_sources: list[str],
    fact_assertions: list[str],
    k: int = 5,
) -> dict:
    """计算单个 case 的评测指标

    Returns:
        {hit_at_k, precision_at_k, mrr, first_hit_rank}
    """
    # 合并所有结果（code + facts）为统一排名列表
    ranked_items: list[tuple[str, str]] = []  # (type, text)
    # 交错合并：code[0], fact[0], code[1], fact[1], ...
    max_len = max(len(code_sources), len(fact_assertions))
    for i in range(max_len):
        if i < len(code_sources):
            ranked_items.append(("code", code_sources[i]))
        if i < len(fact_assertions):
            ranked_items.append(("fact", fact_assertions[i]))

    hits = []
    for rank, (item_type, text) in enumerate(ranked_items[:k], 1):
        is_hit = False
        if item_type == "code" and case.expected_code:
            is_hit = _check_code_hit(text, case.expected_code)
        if item_type == "fact" and case.expected_facts:
            is_hit = _check_fact_hit(text, case.expected_facts)
        hits.append(is_hit)

    hit_at_k = any(hits)
    precision_at_k = sum(hits) / k if hits else 0.0
    first_hit_rank = next((i + 1 for i, h in enumerate(hits) if h), 0)
    mrr = 1.0 / first_hit_rank if first_hit_rank > 0 else 0.0

    return {
        "hit_at_k": hit_at_k,
        "precision_at_k": round(precision_at_k, 3),
        "mrr": round(mrr, 3),
        "first_hit_rank": first_hit_rank,
        "code_count": len(code_sources),
        "fact_count": len(fact_assertions),
    }


# ── 单条评测 ──


async def _run_single_eval(
    case: EvalCase,
    config: Config,
    corpus_db: Database,
    cog_db: object,
    embedder: object,
) -> dict:
    """运行单条查询，返回指标"""
    from index1.unified_search import unified_recall

    result = await unified_recall(
        query=case.query,
        config=config,
        db=corpus_db,
        embedder=embedder,
        db_cog=cog_db,
        collection="index1",
        top_k=10,
    )

    code_sources = [item.source or "" for item in result.code]
    fact_assertions = [item.assertion for item in result.facts]

    metrics = compute_metrics(case, code_sources, fact_assertions, k=5)
    metrics["query"] = case.query
    metrics["query_type"] = case.query_type
    metrics["mode"] = result.mode
    metrics["query_ms"] = result.query_ms

    return metrics


# ── Pytest 测试 ──


class TestEvalBaseline:
    """搜索质量基线测试 — 记录当前指标，不硬性 assert"""

    def test_all_cases(self, config, corpus_db, cog_db, embedder, capsys):
        """运行全部 eval cases，输出汇总报告"""
        results = []
        for case in ALL_CASES:
            metrics = asyncio.run(
                _run_single_eval(case, config, corpus_db, cog_db, embedder)
            )
            results.append(metrics)

        # 按类型分组汇总
        type_groups: dict[str, list[dict]] = {}
        for r in results:
            qt = r["query_type"]
            type_groups.setdefault(qt, []).append(r)

        report_lines = [
            "",
            "=" * 70,
            "  搜索质量评测报告 (Eval Baseline)",
            "=" * 70,
            "",
        ]

        overall_hit = 0
        overall_p5 = 0.0
        overall_mrr = 0.0
        total = len(results)

        for qt, group in sorted(type_groups.items()):
            hits = sum(1 for r in group if r["hit_at_k"])
            avg_p5 = sum(r["precision_at_k"] for r in group) / len(group)
            avg_mrr = sum(r["mrr"] for r in group) / len(group)
            avg_ms = sum(r["query_ms"] for r in group) / len(group)

            report_lines.append(f"  [{qt}] ({len(group)} queries)")
            report_lines.append(f"    Hit@5: {hits}/{len(group)} ({hits/len(group)*100:.0f}%)")
            report_lines.append(f"    P@5:   {avg_p5:.3f}")
            report_lines.append(f"    MRR:   {avg_mrr:.3f}")
            report_lines.append(f"    Avg latency: {avg_ms:.0f}ms")

            # 失败的 case
            failures = [r for r in group if not r["hit_at_k"]]
            if failures:
                report_lines.append(f"    Misses:")
                for f in failures:
                    report_lines.append(f"      - \"{f['query']}\" (code={f['code_count']}, facts={f['fact_count']})")

            report_lines.append("")

            overall_hit += hits
            overall_p5 += avg_p5 * len(group)
            overall_mrr += avg_mrr * len(group)

        report_lines.append("-" * 70)
        report_lines.append(f"  OVERALL ({total} queries)")
        report_lines.append(f"    Hit@5:  {overall_hit}/{total} ({overall_hit/total*100:.0f}%)")
        report_lines.append(f"    P@5:    {overall_p5/total:.3f}")
        report_lines.append(f"    MRR:    {overall_mrr/total:.3f}")
        report_lines.append("-" * 70)
        report_lines.append("")

        report = "\n".join(report_lines)

        # 输出到 stdout（pytest -s 可见）
        with capsys.disabled():
            print(report)

        # 保存到文件
        eval_dir = Path(__file__).parent
        report_file = eval_dir / "baseline_report.txt"
        report_file.write_text(report, encoding="utf-8")

        # 保存 JSON 数据
        json_file = eval_dir / "baseline_data.json"
        json_file.write_text(
            json.dumps(results, ensure_ascii=False, indent=2),
            encoding="utf-8",
        )

        # 软性断言：至少 50% Hit@5（基线门槛，后续逐步提高）
        hit_rate = overall_hit / total
        assert hit_rate >= 0.3, (
            f"Hit@5 过低: {hit_rate:.1%} (期望 ≥30%)\n"
            f"查看详情: {report_file}"
        )

    def test_symbol_lookup_precision(self, config, corpus_db, cog_db, embedder):
        """symbol_lookup 类型应该有高精度"""
        from .eval_cases import SYMBOL_LOOKUP_CASES

        hits = 0
        for case in SYMBOL_LOOKUP_CASES:
            metrics = asyncio.run(
                _run_single_eval(case, config, corpus_db, cog_db, embedder)
            )
            if metrics["hit_at_k"]:
                hits += 1

        hit_rate = hits / len(SYMBOL_LOOKUP_CASES)
        # symbol lookup 应该至少 60% hit（文件名/函数名搜索）
        assert hit_rate >= 0.5, (
            f"symbol_lookup Hit@5: {hit_rate:.1%} ({hits}/{len(SYMBOL_LOOKUP_CASES)})"
        )

    def test_cognition_recall(self, config, corpus_db, cog_db, embedder):
        """cognition_specific 类型应该返回相关 facts"""
        from .eval_cases import COGNITION_CASES

        hits = 0
        for case in COGNITION_CASES:
            metrics = asyncio.run(
                _run_single_eval(case, config, corpus_db, cog_db, embedder)
            )
            if metrics["hit_at_k"]:
                hits += 1

        hit_rate = hits / len(COGNITION_CASES)
        # cognition 至少 30% hit（中文查英文有天然难度）
        assert hit_rate >= 0.2, (
            f"cognition Hit@5: {hit_rate:.1%} ({hits}/{len(COGNITION_CASES)})"
        )
