#!/usr/bin/env python3
"""Leaderboard pre-production test — 3 prompts × 5 models.

Runs a realistic leaderboard bench with aesthetic metric enabled,
then generates both standard HTML report and leaderboard HTML.

Expected cost: ~$0.15-0.25 (generation + judge + metrics)
"""

import os
import sys
import time
from pathlib import Path

# Load .env
env_file = Path(__file__).resolve().parent.parent.parent / ".env"
if env_file.exists():
    for line in env_file.read_text().splitlines():
        line = line.strip()
        if line and not line.startswith("#") and "=" in line:
            k, v = line.split("=", 1)
            os.environ.setdefault(k.strip(), v.strip())

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from evalytic.bench.runner import run_bench
from evalytic.report.terminal import print_full_report
from evalytic.report.html_report import write_html
from evalytic.report.json_report import write_json
from evalytic.report.leaderboard_report import enrich_report, write_leaderboard

# ---------- Config ----------

MODELS = [
    "flux-schnell",       # BFL, $0.003, fast
    "flux-2-dev",         # BFL, $0.025
    "z-image-turbo",      # Other, $0.005
    "ideogram-v3",        # Ideogram, $0.080
    "nano-banana",        # Google (Gemini), $0.005
]

PROMPTS = [
    {"prompt": "A glowing neon sign reading EVALYTIC in a dark rainy alley, reflections on wet cobblestones, cinematic lighting"},
    {"prompt": "A black cat sitting on a stack of old books in a cozy library, warm golden light from a desk lamp, dust particles in the air"},
    {"prompt": "A vintage coffee shop storefront with hand-lettered chalkboard menu, morning sunlight, 35mm film photography style"},
]

# ---------- Pre-flight ----------

print("=== Leaderboard Test — 3 prompts × 5 models ===")
print(f"Models: {', '.join(MODELS)}")
print(f"Prompts: {len(PROMPTS)}")
print(f"Metrics: clip + sharpness + aesthetic")
print(f"Judge: Consensus (gemini-2.5-flash + gpt-5.2)")
print(f"FAL_KEY: {'set' if os.environ.get('FAL_KEY') else 'MISSING!'}")
print(f"GEMINI_API_KEY: {'set' if os.environ.get('GEMINI_API_KEY') else 'MISSING!'}")
print()

# ---------- Run ----------

start = time.monotonic()

report = run_bench(
    models=MODELS,
    prompts=PROMPTS,
    seed=42,
    judges=["gemini-2.5-flash", "gpt-5.2"],
    metrics=["clip", "sharpness", "aesthetic"],
    concurrency=8,
    timeout=600,
    name="leaderboard-test-3x5",
    verbose=True,
)

elapsed = time.monotonic() - start

# ---------- Terminal output ----------

print_full_report(report)

# ---------- Output files ----------

out_dir = Path("scripts/output")
out_dir.mkdir(parents=True, exist_ok=True)

json_path = out_dir / "leaderboard-test-3x5.json"
html_path = out_dir / "leaderboard-test-3x5.html"
lb_path = out_dir / "leaderboard-test-3x5-leaderboard.html"

write_json(report, str(json_path))
print(f"JSON: {json_path.resolve()}")

write_html(report, str(html_path))
print(f"HTML: {html_path.resolve()}")

# ---------- Leaderboard HTML ----------

write_leaderboard(report, str(lb_path))
print(f"Leaderboard HTML: {lb_path.resolve()}")

# ---------- Summary ----------

failures = []
for item in report.items:
    for model, ir in item.results.items():
        if ir.status == "failed":
            failures.append((model, ir.error[:100]))

if failures:
    print(f"\nFAILED ({len(failures)}):")
    for m, err in failures:
        print(f"  {m}: {err}")

total_cost = report.cost.generation_total_usd + report.cost.judge_total_usd
print(f"\nTotal: {elapsed:.0f}s, ${total_cost:.3f}")

# Open leaderboard in browser
import webbrowser
webbrowser.open(f"file://{lb_path.resolve()}")
