"""EB1 Evaluation System.

A comprehensive evaluation framework for comparing routing strategies
across fresh benchmarks with multi-dimensional analysis.

Key components:
- benchmarks: Benchmark loaders and registry
- strategies: Routing/inference strategy wrappers
- metrics: Quality, efficiency, routing, and calibration metrics
- runners: Evaluation orchestration (single and comparison)
- reports: Markdown report generation
- scripts: CLI entry points

Example usage:
    from embergraphrouter.evals import run_evaluation, run_comparison

    # Single strategy evaluation
    results = await run_evaluation(
        strategy="tiered_router:eb1-gnn-preview-12-27",
        benchmark="gpqa_diamond",
        max_examples=50,
    )
    print(f"Accuracy: {results.quality['accuracy'].mean:.3f}")

    # Multi-strategy comparison
    comparison = await run_comparison(
        strategies=["tiered_router", "single_model:gpt-4"],
        benchmark="gpqa_diamond",
    )
    for test in comparison.paired_tests:
        if test.significant:
            print(f"{test.strategy_a} vs {test.strategy_b}: p={test.p_value:.4f}")

CLI usage:
    python -m embergraphrouter.evals.scripts.run_eval \\
        --strategy tiered_router:eb1-gnn-preview-latest \\
        --benchmark gpqa_diamond \\
        --max-examples 50
"""

from embergraphrouter.evals.benchmarks import (
    Benchmark,
    BenchmarkDomain,
    BenchmarkExample,
    EvaluationScore,
    get_benchmark,
    list_benchmarks,
)
from embergraphrouter.evals.metrics import (
    accuracy,
    frac_of_oracle,
    mean_with_ci,
    tier_distribution,
)
from embergraphrouter.evals.reports import (
    ComparisonReport,
    EvaluationReport,
    generate_comparison_report,
    generate_evaluation_report,
)
from embergraphrouter.evals.runners import (
    AggregateMetrics,
    EvaluationResult,
    ExampleResult,
    RunConfig,
)
from embergraphrouter.evals.runners.comparison import (
    ComparisonResult,
    ComparisonRunner,
    run_comparison,
)
from embergraphrouter.evals.runners.simple import (
    SimpleRunner,
    run_evaluation,
)
from embergraphrouter.evals.strategies import (
    CachedResponseStrategy,
    OracleEntry,
    OracleStrategy,
    RandomModelStrategy,
    RoutingDecision,
    SingleModelStrategy,
    Strategy,
    StrategyResult,
    StrategyType,
    TieredRouterStrategy,
)

__all__ = [
    # Benchmarks
    "Benchmark",
    "BenchmarkDomain",
    "BenchmarkExample",
    "EvaluationScore",
    "get_benchmark",
    "list_benchmarks",
    # Strategies
    "CachedResponseStrategy",
    "OracleEntry",
    "OracleStrategy",
    "RandomModelStrategy",
    "RoutingDecision",
    "SingleModelStrategy",
    "Strategy",
    "StrategyResult",
    "StrategyType",
    "TieredRouterStrategy",
    # Runners
    "AggregateMetrics",
    "ComparisonResult",
    "ComparisonRunner",
    "EvaluationResult",
    "ExampleResult",
    "RunConfig",
    "SimpleRunner",
    "run_comparison",
    "run_evaluation",
    # Reports
    "ComparisonReport",
    "EvaluationReport",
    "generate_comparison_report",
    "generate_evaluation_report",
    # Metrics
    "accuracy",
    "frac_of_oracle",
    "mean_with_ci",
    "tier_distribution",
]
