"""
cortex/eval/run_eval.py — Unified cortex-eval CLI entry point (#1282).

Supports ALL benchmarks via a single --benchmark flag:
  - locomo:       LoCoMo memory retrieval (default)
  - locomo-full:  Full native LoCoMo format
  - longmemeval:  LongMemEval long-term memory
  - long-mt-bench: Long-MT-Bench+ conversation continuation

Usage::

    python -m cortex.eval.run_eval --benchmark locomo --sample
    python -m cortex.eval.run_eval --benchmark longmemeval --data path/to/data.json
    python -m cortex.eval.run_eval --benchmark long-mt-bench --data conversations.json
    python -m cortex.eval.run_eval --benchmark locomo --mode rag --config cortex.toml
    python -m cortex.eval.run_eval --help

Results are logged to runs.jsonl for comparison across runs.
"""
from __future__ import annotations

import argparse
import asyncio
import json
import logging
import pathlib
import random
import sys
import time
from typing import Any, Dict, Optional

# Isolation: default owner_id prefix for throwaway benchmark runs
_LOCOMO_EVAL_PREFIX = "locomo-eval"

# Supported benchmarks
BENCHMARKS = ["locomo", "locomo-full", "longmemeval", "long-mt-bench"]


def _build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
        prog="cortex-eval",
        description="Unified Cortex benchmark evaluation CLI (#1282).",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Benchmarks:
  locomo          LoCoMo memory retrieval (internal format)
  locomo-full     Full native LoCoMo format (locomo10.json)
  longmemeval     LongMemEval long-term memory
  long-mt-bench   Long-MT-Bench+ conversation continuation (LLM judge, 1-10)

Examples:
  cortex-eval --benchmark locomo --sample
  cortex-eval --benchmark locomo --config cortex.toml --mode rag
  cortex-eval --benchmark longmemeval --data /path/to/data.json --out results.json
  cortex-eval --benchmark long-mt-bench --data conversations.json
  cortex-eval --benchmark locomo-full --sample-id conv-26
  cortex-eval --benchmark locomo --fail-under 0.75
        """,
    )

    # Required
    p.add_argument(
        "--benchmark", "--dataset",
        required=True,
        choices=BENCHMARKS,
        dest="benchmark",
        help="Benchmark to evaluate against.",
    )

    # Data source
    p.add_argument("--config", default=None, help="Path to cortex.toml config file.")
    p.add_argument("--data", default=None, help="Path to dataset JSON file.")
    p.add_argument("--sample", action="store_true", help="Use bundled sample dataset.")
    p.add_argument("--sample-id", default=None, dest="sample_id",
                    help="For locomo-full: load a single conversation by sample_id.")
    p.add_argument("--db-path", default=None, dest="db_path",
                    help="Path to SQLite DB for eval (uses temp file if omitted).")

    # Eval config
    p.add_argument("--mode", default="retrieval-only",
                    choices=["retrieval-only", "rag", "full"],
                    help="Eval mode (default: retrieval-only).")
    p.add_argument("--fuzzy-threshold", type=float, default=0.6, dest="fuzzy_threshold",
                    help="Fuzzy match threshold (default: 0.6).")
    p.add_argument("--llm-judge", action="store_true", dest="llm_judge",
                    help="Enable LLM-as-judge scoring.")
    p.add_argument("--confidence-threshold", type=float, default=0.3,
                    dest="confidence_threshold",
                    help="Confidence threshold for refusal (#1275, default: 0.3).")

    # Output
    p.add_argument("--out", default=None, help="Write full results JSON to this file.")
    p.add_argument("--runs-log", default=None, dest="runs_log",
                    help="Path to runs.jsonl log file (default: eval_runs.jsonl in cwd).")

    # CI integration
    p.add_argument("--fail-under", type=float, default=None, dest="fail_under",
                    help="Exit 1 if fuzzy_match_accuracy < threshold (CI gate).")
    p.add_argument("--baseline", default=None,
                    help="Path to baseline results JSON for comparison.")

    # Isolation (default ON — prevents eval characters leaking into production DB)
    isolation_group = p.add_mutually_exclusive_group()
    isolation_group.add_argument(
        "--isolate",
        action="store_true",
        default=True,
        dest="isolate",
        help=(
            "Use a dedicated throwaway owner_id (locomo-eval-<timestamp>) so benchmark "
            "entities/claims are isolated from production data.  Auto-cleanup runs after "
            "the benchmark completes.  DEFAULT: on."
        ),
    )
    isolation_group.add_argument(
        "--no-isolate",
        action="store_false",
        dest="isolate",
        help=(
            "Disable isolation — benchmark writes to the plugin's default owner_id.  "
            "Use only for debugging; production data may be contaminated."
        ),
    )

    # Safety
    p.add_argument("--force", action="store_true",
                    help="Skip confirmation prompts (e.g. with --no-isolate).")

    # Verbosity
    p.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging.")

    return p


def _load_plugin(config_path: Optional[str], db_path: Optional[str] = None) -> Any:
    """Load CortexPlugin from config or create a minimal in-memory instance.

    Fixes #1273: Ensures the plugin has an LLM provider wired for extraction.
    """
    try:
        from cortex.api.plugin import CortexPlugin
    except ImportError as e:
        print(f"ERROR: Could not import CortexPlugin: {e}", file=sys.stderr)
        sys.exit(1)

    _temp_db_path = None
    if config_path:
        config_file = pathlib.Path(config_path)
        if not config_file.exists():
            print(f"ERROR: Config file not found: {config_path}", file=sys.stderr)
            sys.exit(1)
        try:
            plugin = CortexPlugin(config_path=config_path, db_path=db_path)
        except Exception as e:
            print(f"ERROR: Failed to init CortexPlugin from config: {e}", file=sys.stderr)
            sys.exit(1)
    else:
        import atexit
        import tempfile
        if db_path is None:
            _tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
            db_path = _tmp.name
            _tmp.close()
            _temp_db_path = db_path
            atexit.register(lambda p=_temp_db_path: pathlib.Path(p).unlink(missing_ok=True))
        try:
            plugin = CortexPlugin(db_path=db_path)
        except Exception as e:
            print(f"ERROR: Failed to create default CortexPlugin: {e}", file=sys.stderr)
            sys.exit(1)

    if getattr(plugin, "llm", None) is None:
        print(
            "⚠️  WARNING: No LLM provider configured. Extraction will produce 0 claims.\n"
            "   Set GOOGLE_API_KEY, OPENROUTER_API_KEY, or ANTHROPIC_API_KEY,\n"
            "   or provide a --config with [cortex.providers] section.",
            file=sys.stderr,
        )

    return plugin


def _build_llm_judge(plugin: Any) -> Optional[Any]:
    """Build an LLM judge callable from the plugin's LLM provider."""
    try:
        llm = getattr(plugin, "llm", None)
        if llm is None:
            print("WARNING: --llm-judge requested but no LLM configured.", file=sys.stderr)
            return None

        async def _judge_async(question: str, prediction: str, reference: str) -> float:
            prompt = (
                f"Question: {question}\nGold answer: {reference}\n"
                f"System answer: {prediction}\n\n"
                "Score correctness from 0.0 to 1.0. Reply with ONLY a number."
            )
            response = await llm.complete(prompt)
            text = str(response).strip()
            try:
                return max(0.0, min(1.0, float(text)))
            except ValueError:
                return 0.0

        def judge(question: str, prediction: str, reference: str) -> float:
            try:
                loop = asyncio.get_event_loop()
                if loop.is_running():
                    import concurrent.futures
                    with concurrent.futures.ThreadPoolExecutor() as ex:
                        future = ex.submit(asyncio.run, _judge_async(question, prediction, reference))
                        return future.result()
                return loop.run_until_complete(_judge_async(question, prediction, reference))
            except Exception:
                return 0.0

        return judge
    except Exception as e:
        print(f"WARNING: LLM judge setup failed: {e}", file=sys.stderr)
        return None


def _log_run(
    runs_log: Optional[str],
    benchmark: str,
    metrics: Dict[str, Any],
    config: Dict[str, Any],
    elapsed_sec: float,
) -> None:
    """Append run result to runs.jsonl log file."""
    import datetime
    log_path = pathlib.Path(runs_log or "eval_runs.jsonl")
    entry = {
        "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
        "benchmark": benchmark,
        "elapsed_sec": round(elapsed_sec, 1),
        "metrics": metrics,
        "config": config,
    }
    try:
        with open(log_path, "a", encoding="utf-8") as f:
            f.write(json.dumps(entry, default=str) + "\n")
    except Exception as exc:
        print(f"WARNING: Failed to write runs log: {exc}", file=sys.stderr)


def _compare_baseline(
    baseline_path: str,
    current_metrics: Dict[str, Any],
) -> None:
    """Compare current metrics against a baseline and print diff."""
    try:
        with open(baseline_path, "r") as f:
            baseline = json.load(f)
        base_metrics = baseline.get("metrics", baseline)

        print("\n📊 Baseline comparison:")
        for key in ["exact_match_accuracy", "fuzzy_match_accuracy", "avg_token_f1"]:
            base_val = base_metrics.get(key, 0.0)
            curr_val = current_metrics.get(key, 0.0)
            diff = curr_val - base_val
            arrow = "↑" if diff > 0 else "↓" if diff < 0 else "="
            print(f"  {key}: {base_val:.3f} → {curr_val:.3f} ({arrow} {diff:+.3f})")
    except Exception as exc:
        print(f"WARNING: Baseline comparison failed: {exc}", file=sys.stderr)


async def _cleanup_eval_owner(plugin: Any, owner_id: str, verbose: bool = False) -> None:
    """Delete all claims and entities written under the throwaway eval owner_id.

    Uses bulk SQL in proper FK order so that no eval characters (Sarah Chen
    etc.) leak into production recall results and no FK constraint errors occur.
    """
    storage = getattr(plugin, "storage", None)
    if storage is None:
        if verbose:
            print(f"  ⚠ Cleanup skipped — plugin has no .storage attribute", file=sys.stderr)
        return

    try:
        conn = await storage._get_conn()
        async with storage._write_lock:
            # 1. Collect claim IDs for this owner
            async with conn.execute(
                "SELECT id FROM semantic_claim WHERE owner_id = ?",
                (owner_id,),
            ) as cur:
                claim_rows = await cur.fetchall()
            claim_ids = [r[0] if not hasattr(r, "keys") else r["id"] for r in claim_rows]

            # 2. Collect entity IDs for this owner
            async with conn.execute(
                "SELECT id FROM entity WHERE owner_id = ?",
                (owner_id,),
            ) as cur:
                entity_rows = await cur.fetchall()
            entity_ids = [r[0] if not hasattr(r, "keys") else r["id"] for r in entity_rows]

            if claim_ids:
                ph = ",".join("?" for _ in claim_ids)

                # --- Delete in FK-safe order (children before parents) ---

                # Embeddings: vec_index (virtual table, may not exist)
                if getattr(storage, "_vec_loaded", False):
                    try:
                        await conn.execute(
                            f"DELETE FROM vec_index WHERE item_type = 'claim' AND item_id IN ({ph})",
                            claim_ids,
                        )
                    except Exception:
                        pass  # vec_index may not exist; non-fatal

                # Embeddings: embedding_index
                await conn.execute(
                    f"DELETE FROM embedding_index WHERE item_type = 'claim' AND item_id IN ({ph})",
                    claim_ids,
                )

                # FTS
                await conn.execute(
                    f"DELETE FROM fts_claims WHERE claim_id IN ({ph})",
                    claim_ids,
                )

                # Provenance (FK ON DELETE CASCADE, but be explicit)
                await conn.execute(
                    f"DELETE FROM claim_provenance WHERE claim_id IN ({ph})",
                    claim_ids,
                )

                # Deprecation log
                await conn.execute(
                    f"DELETE FROM deprecation_log WHERE claim_id IN ({ph})",
                    claim_ids,
                )

                # Claim changelog (v6)
                try:
                    await conn.execute(
                        f"DELETE FROM claim_changelog WHERE claim_id IN ({ph})",
                        claim_ids,
                    )
                except Exception:
                    pass  # table may not exist in older schemas

                # Memory activation (v8, ON DELETE CASCADE)
                try:
                    await conn.execute(
                        f"DELETE FROM memory_activation WHERE claim_id IN ({ph})",
                        claim_ids,
                    )
                except Exception:
                    pass

                # Memory evidence (v8, ON DELETE CASCADE)
                try:
                    await conn.execute(
                        f"DELETE FROM memory_evidence WHERE claim_id IN ({ph})",
                        claim_ids,
                    )
                except Exception:
                    pass

                # Memory edges referencing these claims
                await conn.execute(
                    f"DELETE FROM memory_edge WHERE (source_type = 'claim' AND source_id IN ({ph})) "
                    f"OR (target_type = 'claim' AND target_id IN ({ph}))",
                    claim_ids + claim_ids,
                )

                # Clear self-referential superseded_by before deleting claims
                await conn.execute(
                    f"UPDATE semantic_claim SET superseded_by = NULL "
                    f"WHERE owner_id = ? AND superseded_by IN ({ph})",
                    [owner_id] + claim_ids,
                )

                # Finally: delete the claims themselves
                await conn.execute(
                    "DELETE FROM semantic_claim WHERE owner_id = ?",
                    (owner_id,),
                )

            if entity_ids:
                eph = ",".join("?" for _ in entity_ids)

                # Entity aliases (FK ON DELETE CASCADE, but be explicit)
                await conn.execute(
                    f"DELETE FROM entity_alias WHERE entity_id IN ({eph})",
                    entity_ids,
                )

                # Peer references (v7, may not exist)
                try:
                    await conn.execute(
                        f"DELETE FROM peer WHERE entity_id IN ({eph})",
                        entity_ids,
                    )
                except Exception:
                    pass

                # Entities
                await conn.execute(
                    "DELETE FROM entity WHERE owner_id = ?",
                    (owner_id,),
                )

            await conn.commit()

        deleted_claims = len(claim_ids) if claim_ids else 0
        deleted_entities = len(entity_ids) if entity_ids else 0

        if verbose or deleted_claims > 0 or deleted_entities > 0:
            print(
                f"  Cleanup [{owner_id}]: removed {deleted_claims} claims, "
                f"{deleted_entities} entities",
                file=sys.stderr,
            )

    except Exception as exc:
        print(f"  ⚠ Cleanup failed: {exc}", file=sys.stderr)
        if verbose:
            import traceback
            traceback.print_exc(file=sys.stderr)


async def _run(args: argparse.Namespace) -> int:
    """Main async eval runner. Returns exit code."""
    t0 = time.perf_counter()

    # Long-MT-Bench+ uses a different evaluation path
    if args.benchmark == "long-mt-bench":
        return await _run_long_mt_bench(args, t0)

    # Memory benchmarks (locomo, longmemeval)
    from .harness import EvalHarness

    plugin = _load_plugin(args.config, db_path=args.db_path)

    # Ensure SQLiteAdapter is initialized (required for eval — normally
    # done by http_server.py at FastAPI startup).  The adapter has no
    # _initialized flag; _conn is None until initialize() is called.
    if hasattr(plugin, "storage") and plugin.storage is not None:
        if getattr(plugin.storage, "_conn", None) is None:
            await plugin.storage.initialize()

    # B-LC5: Pre-flight validation for dataset file
    data_path = args.data if args.data else None
    if data_path and not pathlib.Path(data_path).exists():
        print(f"ERROR: Dataset file not found: {data_path}", file=sys.stderr)
        return 1

    # Resolve owner_id — isolation is ON by default (#1177)
    isolate: bool = getattr(args, "isolate", True)
    eval_owner_id: Optional[str] = None
    if isolate:
        ts = int(time.time())
        # B-LC3: Add random entropy to avoid 1-second timestamp collisions
        eval_owner_id = f"{_LOCOMO_EVAL_PREFIX}-{ts}-{random.randint(1000, 9999)}"
        print(
            f"ℹ️  Isolation ON: using throwaway owner_id '{eval_owner_id}' "
            f"(pass --no-isolate to disable)",
            file=sys.stderr,
        )
    else:
        # B-LC4: Confirmation gate for --no-isolate
        if not getattr(args, "force", False):
            if sys.stdin.isatty():
                print(
                    "⚠️  Isolation OFF (--no-isolate): benchmark will write to the "
                    "plugin default owner_id.  Production data may be contaminated.",
                    file=sys.stderr,
                )
                try:
                    answer = input("Continue without isolation? [y/N] ").strip().lower()
                except (EOFError, KeyboardInterrupt):
                    answer = ""
                if answer not in ("y", "yes"):
                    print("Aborted.", file=sys.stderr)
                    return 1
            else:
                print(
                    "ERROR: --no-isolate requires --force in non-interactive mode.",
                    file=sys.stderr,
                )
                return 1
        else:
            print(
                "⚠️  Isolation OFF (--no-isolate --force): benchmark will write to the "
                "plugin default owner_id.  Production data may be contaminated.",
                file=sys.stderr,
            )

    llm_judge = None
    if args.llm_judge:
        llm_judge = _build_llm_judge(plugin)

    rag_llm = None
    if args.mode in ("rag", "full"):
        rag_llm = getattr(plugin, "llm", None)
        if rag_llm is None:
            print("WARNING: --mode rag/full but no LLM configured.", file=sys.stderr)

    harness = EvalHarness(
        plugin=plugin,
        fuzzy_threshold=args.fuzzy_threshold,
        llm_judge=llm_judge,
        mode=args.mode,
        llm=rag_llm,
        confidence_threshold=args.confidence_threshold,
        owner_id=eval_owner_id,
    )

    # B-LC1: Wrap eval execution in try/finally to guarantee cleanup on crash
    try:
        if args.benchmark == "locomo":
            report = await harness.run_locomo(path=data_path)
        elif args.benchmark == "locomo-full":
            from .locomo import LoCoMoLoader
            loader = LoCoMoLoader()
            if args.sample_id is not None:
                dataset = loader.load_conversation(args.sample_id, path=data_path)
            else:
                dataset = loader.load_native(path=data_path)
            report = await harness.run_locomo_dataset(dataset)
        elif args.benchmark == "longmemeval":
            report = await harness.run_longmemeval(path=data_path)
        else:
            print(f"ERROR: Unknown benchmark: {args.benchmark}", file=sys.stderr)
            return 1

        elapsed = time.perf_counter() - t0
        print(f"\nEval completed in {elapsed:.1f}s")
        print(report.summary())

        # Log run
        _log_run(args.runs_log, args.benchmark, report.metrics,
                 report.config, elapsed)

        # Write results
        if args.out:
            out_path = pathlib.Path(args.out)
            with open(out_path, "w", encoding="utf-8") as f:
                json.dump(report.to_dict(), f, indent=2, default=str)
            print(f"\nResults written to: {out_path}")

        # Baseline comparison
        if args.baseline:
            _compare_baseline(args.baseline, report.metrics)

        # CI fail-under gate
        exit_code = 0
        if args.fail_under is not None:
            accuracy = report.metrics.get("fuzzy_match_accuracy", 0.0)
            if accuracy < args.fail_under:
                print(
                    f"\nFAIL: fuzzy_match_accuracy {accuracy:.1%} < {args.fail_under:.1%}",
                    file=sys.stderr,
                )
                exit_code = 1
            else:
                print(f"\nPASS: fuzzy_match_accuracy {accuracy:.1%} >= {args.fail_under:.1%}")

        return exit_code

    finally:
        # Isolation cleanup — remove throwaway eval entities/claims (#1177)
        # B-LC1: Always runs, even if eval crashes mid-execution
        if isolate and eval_owner_id:
            print(f"\nCleaning up eval data for owner '{eval_owner_id}'...", file=sys.stderr)
            await _cleanup_eval_owner(plugin, eval_owner_id, verbose=args.verbose)


async def _run_long_mt_bench(args: argparse.Namespace, t0: float) -> int:
    """Run Long-MT-Bench+ evaluation (#1281)."""
    from .long_mt_bench import LongMTBenchAdapter, LongMTBenchConfig, load_long_mt_bench

    plugin = _load_plugin(args.config, db_path=args.db_path)
    llm = getattr(plugin, "llm", None)
    if llm is None:
        print("ERROR: Long-MT-Bench+ requires an LLM provider.", file=sys.stderr)
        return 1

    conversations = load_long_mt_bench(args.data)
    adapter = LongMTBenchAdapter(llm=llm)
    report = await adapter.evaluate(conversations)

    elapsed = time.perf_counter() - t0
    print(f"\nEval completed in {elapsed:.1f}s")
    print(report.summary())

    # Log run
    _log_run(args.runs_log, "long-mt-bench", report.metrics, {
        "max_history_tokens": adapter.config.max_history_tokens,
        "judge_model": adapter.config.judge_model,
        "generation_model": adapter.config.generation_model,
    }, elapsed)

    if args.out:
        out_path = pathlib.Path(args.out)
        with open(out_path, "w", encoding="utf-8") as f:
            json.dump(report.to_dict(), f, indent=2, default=str)
        print(f"\nResults written to: {out_path}")

    # Baseline comparison
    if args.baseline:
        _compare_baseline(args.baseline, report.metrics)

    # CI gate: for MT-Bench, fail-under is on avg_score (1-10 scale)
    if args.fail_under is not None:
        score = report.metrics.get("avg_score", 0.0)
        if score < args.fail_under:
            print(f"\nFAIL: avg_score {score:.2f} < {args.fail_under:.2f}", file=sys.stderr)
            return 1
        print(f"\nPASS: avg_score {score:.2f} >= {args.fail_under:.2f}")

    return 0


def main() -> None:
    parser = _build_parser()
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(name)s: %(message)s")
    else:
        logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s")

    exit_code = asyncio.run(_run(args))
    sys.exit(exit_code)


if __name__ == "__main__":
    main()
