#!/usr/bin/env python3
"""Leaderboard pilot — configurable model count.

Usage:
  python scripts/leaderboard_pilot.py              # 2-model smoke test (~$0.01)
  python scripts/leaderboard_pilot.py --full       # all 29 working models (~$1.40 + judge)
"""

import argparse
import os
import sys
import time
from pathlib import Path

# Load .env
env_file = Path(__file__).resolve().parent.parent.parent / ".env"
if env_file.exists():
    for line in env_file.read_text().splitlines():
        line = line.strip()
        if line and not line.startswith("#") and "=" in line:
            k, v = line.split("=", 1)
            os.environ.setdefault(k.strip(), v.strip())

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from evalytic.bench.runner import run_bench
from evalytic.bench.registry import MODEL_REGISTRY
from evalytic.report.terminal import print_full_report
from evalytic.report.html_report import write_html
from evalytic.report.json_report import write_json

# ---------- Model selection ----------

EXCLUDE = {"flux-dev", "sdxl", "sd3", "recraft-v4-pro", "hidream", "hunyuan-3", "glm-image", "sd35-large"}
ALL_MODELS = [k for k, v in MODEL_REGISTRY.items() if v.pipeline == "text2img" and k not in EXCLUDE]
SMOKE_MODELS = ["flux-schnell", "z-image-turbo"]  # cheapest two: $0.003 + $0.005

PROMPT = "A glowing neon sign reading EVALYTIC in a dark rainy alley, reflections on wet cobblestones, cinematic lighting"

# ---------- Args ----------

parser = argparse.ArgumentParser()
parser.add_argument("--full", action="store_true", help="Run all 29 models instead of 2-model smoke test")
args = parser.parse_args()

models = ALL_MODELS if args.full else SMOKE_MODELS
mode = "FULL (29 models)" if args.full else "SMOKE TEST (2 models)"

est_gen = sum(MODEL_REGISTRY[m].cost_per_image for m in models)
est_judge = len(models) * 3 * 0.001  # ~$0.001 per judge call, 3 dims
est_total = est_gen + est_judge

print(f"=== Leaderboard Pilot — {mode} ===")
print(f"Models: {len(models)}")
print(f"Judge: gemini-2.5-flash (3 dims)")
print(f"Est cost: ${est_total:.3f} (gen: ${est_gen:.3f} + judge: ${est_judge:.3f})")
print(f"FAL_KEY: {'set' if os.environ.get('FAL_KEY') else 'MISSING!'}")
print(f"GEMINI_API_KEY: {'set' if os.environ.get('GEMINI_API_KEY') else 'MISSING!'}")
print()

# ---------- Run ----------

start = time.monotonic()

report = run_bench(
    models=models,
    prompts=[{"prompt": PROMPT}],
    seed=42,
    judge="gemini-2.5-flash",
    metrics=["clip", "sharpness"],
    concurrency=8,
    timeout=600,
    name=f"leaderboard-pilot-{'full' if args.full else 'smoke'}",
    verbose=True,
)

elapsed = time.monotonic() - start

# ---------- Terminal output ----------

print_full_report(report)

# ---------- JSON + HTML ----------

out_dir = Path("scripts/output")
out_dir.mkdir(parents=True, exist_ok=True)

suffix = "full" if args.full else "smoke"
json_path = out_dir / f"leaderboard-{suffix}.json"
html_path = out_dir / f"leaderboard-{suffix}.html"

write_json(report, str(json_path))
print(f"JSON: {json_path.resolve()}")

write_html(report, str(html_path))
print(f"HTML: {html_path.resolve()}")

# ---------- Failures ----------

failures = []
for item in report.items:
    for model, ir in item.results.items():
        if ir.status == "failed":
            failures.append((model, ir.error[:100]))

if failures:
    print(f"\nFAILED ({len(failures)}):")
    for m, err in failures:
        print(f"  {m}: {err}")

print(f"\nTotal: {elapsed:.0f}s, ${report.cost.generation_total_usd + report.cost.judge_total_usd:.3f}")
