#!/usr/bin/env python
"""Run evaluation of a strategy against a benchmark.

Example usage:
    python -m embergraphrouter.evals.scripts.run_eval \\
        --strategy tiered_router:eb1-gnn-preview-latest \\
        --benchmark gpqa_diamond \\
        --max-examples 50 \\
        --output results/gpqa_eval.json

    python -m embergraphrouter.evals.scripts.run_eval \\
        --strategy single_model:gpt-4 \\
        --benchmark simpleqa \\
        --max-examples 100
"""

import argparse
import asyncio
import logging
import sys
from datetime import datetime
from pathlib import Path


def main() -> int:
    """Main entry point for evaluation CLI."""
    parser = argparse.ArgumentParser(
        description="Run evaluation of a strategy against a benchmark.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Strategy formats:
  tiered_router                     TieredRouter with default registry
  tiered_router:eb1-gnn-preview-latest   TieredRouter with specific registry
  single_model:gpt-4                Single model baseline
  random:gpt-4,claude-opus-4.5,gemini-3-pro  Random selection baseline

Available benchmarks:
  gpqa_diamond      GPQA graduate-level QA (fresh)
  simpleqa          SimpleQA factoid QA (fresh)
  humaneval         HumanEval code generation (fresh)
""",
    )

    parser.add_argument(
        "--strategy",
        "-s",
        type=str,
        default=None,
        help="Strategy specification (e.g., tiered_router:eb1-gnn-preview-latest)",
    )
    parser.add_argument(
        "--benchmark",
        "-b",
        type=str,
        default=None,
        help="Benchmark name (e.g., gpqa_diamond, simpleqa)",
    )
    parser.add_argument(
        "--max-examples",
        "-n",
        type=int,
        default=None,
        help="Maximum examples to evaluate (default: all)",
    )
    parser.add_argument(
        "--subset",
        type=str,
        default=None,
        help="Benchmark subset (e.g., test, validation)",
    )
    parser.add_argument(
        "--system-prompt",
        type=str,
        default=None,
        help="System prompt to use for all queries",
    )
    parser.add_argument(
        "--output",
        "-o",
        type=str,
        default=None,
        help="Output path for JSON results",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        help="Enable verbose logging",
    )
    parser.add_argument(
        "--list-benchmarks",
        action="store_true",
        help="List available benchmarks and exit",
    )
    parser.add_argument(
        "--list-fresh",
        action="store_true",
        help="List only fresh (OOD) benchmarks and exit",
    )

    args = parser.parse_args()

    # Configure logging
    log_level = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(
        level=log_level,
        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )
    logger = logging.getLogger(__name__)

    # Handle list commands
    if args.list_benchmarks or args.list_fresh:
        from embergraphrouter.evals.benchmarks import list_benchmarks
        benchmarks = list_benchmarks(fresh_only=args.list_fresh)
        label = "Fresh benchmarks" if args.list_fresh else "Available benchmarks"
        print(f"\n{label}:")
        for b in benchmarks:
            print(f"  - {b}")
        return 0

    # Validate required args for evaluation
    if not args.strategy:
        parser.error("--strategy is required for evaluation")
    if not args.benchmark:
        parser.error("--benchmark is required for evaluation")

    # Generate default output path
    output_path = args.output
    if output_path is None:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        output_dir = Path("results/evals")
        output_dir.mkdir(parents=True, exist_ok=True)
        strategy_slug = args.strategy.replace(":", "_")
        output_path = output_dir / f"{args.benchmark}_{strategy_slug}_{timestamp}.json"

    logger.info("Starting evaluation")
    logger.info("  Strategy: %s", args.strategy)
    logger.info("  Benchmark: %s", args.benchmark)
    logger.info("  Max examples: %s", args.max_examples or "all")
    logger.info("  Output: %s", output_path)

    try:
        from embergraphrouter.evals.runners.simple import run_evaluation

        result = asyncio.run(
            run_evaluation(
                strategy=args.strategy,
                benchmark=args.benchmark,
                max_examples=args.max_examples,
                system_prompt=args.system_prompt,
                subset=args.subset,
                output_path=output_path,
            )
        )

        # Print summary
        print("\n" + "=" * 60)
        print("EVALUATION SUMMARY")
        print("=" * 60)
        print(f"Strategy:    {result.strategy_name}")
        print(f"Benchmark:   {result.benchmark_name}")
        print(f"Examples:    {result.n_examples}")
        print(f"Errors:      {len(result.errors)}")
        print()

        if result.quality:
            print("Quality Metrics:")
            for metric_name, metrics in result.quality.items():
                print(
                    f"  {metric_name}: {metrics.mean:.4f} "
                    f"[{metrics.ci_lower:.4f}, {metrics.ci_upper:.4f}]"
                )

        if result.efficiency:
            print("\nEfficiency Metrics:")
            for metric_name, metrics in result.efficiency.items():
                print(f"  {metric_name}: {metrics.mean:.4f}")

        if result.routing:
            print("\nRouting Metrics:")
            if "tier_distribution" in result.routing:
                print("  Tier distribution:")
                for tier, pct in result.routing["tier_distribution"].items():
                    print(f"    {tier}: {pct:.1%}")

        print()
        print(f"Results saved to: {output_path}")
        print("=" * 60)

        return 0

    except Exception as e:
        logger.exception("Evaluation failed: %s", e)
        return 1


if __name__ == "__main__":
    sys.exit(main())
