"""Experiment agent tools — trace replay, A/B testing, auto-evaluation, response comparison."""
from __future__ import annotations

import json
import logging
from concurrent.futures import ThreadPoolExecutor, as_completed
from statistics import mean
from typing import Any

from src.runtime_config import load_runtime_profile
from src.tools.analysis import (
    _auto_score_response,
    _compare_responses_with_llm,
    _comparison_model_specs,
    _parse_model_spec,
    _replay_trace,
)
from src.tools.langfuse_fetch import (
    _build_langfuse_client,
    _fetch_trace_detail,
)
from src.user_store import (
    create_experiment,
    create_experiment_run,
    get_experiment,
    list_experiment_runs,
    list_experiments,
    update_experiment_run,
    update_experiment_status,
)

try:
    from langchain_core.tools import tool as _lc_tool
except Exception:
    def _lc_tool(*_args, **_kwargs):
        def _decorator(fn):
            return fn
        return _decorator

_log = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Shared helpers (C1/C2/S3)
# ---------------------------------------------------------------------------

def _get_profile() -> dict[str, Any]:
    try:
        return load_runtime_profile() or {}
    except Exception:
        return {}


def _resolve_user_id(user_id: str) -> str:
    """Return a meaningful user_id — prefer profile value over default 'agent'."""
    if user_id and user_id != "agent":
        return user_id
    profile = _get_profile()
    return str(profile.get("user_id") or profile.get("username") or user_id or "agent")


def _to_dict(obj: Any) -> dict[str, Any]:
    """Convert a Langfuse object / dict / unknown to plain dict."""
    if isinstance(obj, dict):
        return obj
    if hasattr(obj, "__dict__"):
        return obj.__dict__
    return {"raw": str(obj)}


def _extract_text(value: Any) -> str:
    """Extract a plain text string from a trace input/output value."""
    if value is None:
        return ""
    if isinstance(value, dict):
        return str(value.get("content", "") or json.dumps(value, ensure_ascii=False))
    if isinstance(value, list):
        return json.dumps(value, ensure_ascii=False)
    return str(value)


def _default_judge_spec(profile: dict[str, Any]) -> dict[str, Any]:
    """Return default judge model spec (prefer gpt-4o, fallback to first available)."""
    specs = _comparison_model_specs(profile)
    for s in specs:
        if "gpt-4" in str(s.get("model", "")):
            return s
    return specs[0] if specs else {"provider": "openai", "model": "gpt-4o", "alias": "gpt-4o"}


def _build_client_safe() -> tuple[Any, Any, str | None]:
    try:
        return _build_langfuse_client({})
    except Exception as e:
        return None, None, str(e)


def _err(msg: str) -> str:
    return json.dumps({"error": msg}, ensure_ascii=False)


def _json_safe_list(raw: str | list) -> list:
    if isinstance(raw, list):
        return raw
    try:
        return json.loads(raw)
    except (json.JSONDecodeError, TypeError):
        return []


def _json_safe_dict(raw: str | dict) -> dict:
    if isinstance(raw, dict):
        return raw
    try:
        return json.loads(raw)
    except (json.JSONDecodeError, TypeError):
        return {}


# ---------------------------------------------------------------------------
# Tool 1: Trace Replay
# ---------------------------------------------------------------------------

@_lc_tool("trace_replay_execution")
def trace_replay_tool(
    trace_id: str,
    model_specs: list[str],
    prompt_override: str = "",
    auto_score: bool = True,
    experiment_name: str = "",
    user_id: str = "agent",
    project_id: str = "",
) -> str:
    """Replay a specific trace with different models/prompts and compare results.
    Fetches the original trace, re-executes it with each specified model,
    optionally auto-scores the outputs, and saves results as an experiment.

    Args:
        trace_id: The Langfuse trace ID to replay.
        model_specs: List of model specs (e.g. ["openai:gpt-4o", "anthropic:claude-sonnet-4-20250514"]).
        prompt_override: Optional custom prompt to use instead of the original.
        auto_score: Whether to automatically score outputs using LLM-as-Judge (default True).
        experiment_name: Optional name for the experiment.
        user_id: User ID for tracking.
        project_id: Optional project ID.
    """
    profile = _get_profile()
    resolved_uid = _resolve_user_id(user_id)
    client, _cfg, err = _build_client_safe()
    if err:
        return _err(f"Langfuse connection failed: {err}")

    # Fetch original trace
    try:
        trace = _fetch_trace_detail(client, trace_id)
        if not trace:
            return _err(f"Trace {trace_id} not found")
        trace_data = _to_dict(trace)
    except Exception as e:
        return _err(f"Failed to fetch trace: {e}")

    # Parse model specs
    parsed_specs = []
    for raw in model_specs:
        try:
            parsed_specs.append(_parse_model_spec(raw))
        except Exception as e:
            return _err(f"Invalid model spec '{raw}': {e}")

    # Create experiment
    exp_name = experiment_name or f"replay_{trace_id[:8]}"
    config = {
        "trace_id": trace_id,
        "models": model_specs,
        "auto_score": auto_score,
        "prompt_override": prompt_override or None,
    }
    exp_id = create_experiment(
        user_id=resolved_uid,
        name=exp_name,
        experiment_type="replay",
        config=config,
        description=f"Trace replay: {trace_id}",
        project_id=project_id or None,
    )

    try:
        update_experiment_status(exp_id, "running")

        input_text = _extract_text(trace_data.get("input"))
        original_output = _extract_text(trace_data.get("output"))

        results = {}
        prompt_text = prompt_override or None

        def _run_replay(spec: dict) -> tuple[str, dict]:
            label = str(spec.get("alias", spec.get("model", "unknown")))
            try:
                replay_result = _replay_trace(
                    profile, trace_data, spec,
                    prompt_override=prompt_override or None,
                )
                run_id = create_experiment_run(
                    experiment_id=exp_id,
                    variant_label=label,
                    model_spec=f"{spec.get('provider', '')}:{spec.get('model', '')}",
                    input_text=input_text,
                    prompt_text=prompt_text,
                    trace_id=trace_id,
                )

                scores: dict = {}
                if auto_score and replay_result.get("output"):
                    judge = _default_judge_spec(profile)
                    scores = _auto_score_response(
                        profile, judge, input_text, str(replay_result["output"])
                    )

                update_experiment_run(
                    run_id,
                    output_text=str(replay_result.get("output", "")),
                    scores_json=scores,
                    latency_ms=replay_result.get("latency_ms"),
                    status="completed",
                )
                return label, {
                    "output": str(replay_result.get("output", ""))[:500],
                    "latency_ms": replay_result.get("latency_ms"),
                    "scores": scores,
                }
            except Exception as e:
                _log.warning("Replay failed for %s: %s", label, e)
                return label, {"error": str(e)}

        max_workers = min(len(parsed_specs), 5)
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            futures = {executor.submit(_run_replay, spec): spec for spec in parsed_specs}
            for future in as_completed(futures):
                label, result = future.result()
                results[label] = result

        update_experiment_status(exp_id, "completed", result_json=results)

        summary = {
            "experiment_id": exp_id,
            "trace_id": trace_id,
            "original_output": str(original_output)[:300] if original_output else None,
            "variants": results,
        }
        return json.dumps(summary, ensure_ascii=False, indent=2)

    except Exception as e:
        _log.exception("trace_replay_tool failed for exp %s", exp_id)
        update_experiment_status(exp_id, "failed", result_json={"error": str(e)})
        return _err(f"Experiment failed: {e}")


# ---------------------------------------------------------------------------
# Tool 2: Experiment Manager
# ---------------------------------------------------------------------------

@_lc_tool("experiment_manager")
def experiment_manager_tool(
    action: str,
    experiment_id: int = 0,
    name: str = "",
    dataset_name: str = "",
    variants: str = "[]",
    max_items: int = 50,
    auto_score: bool = True,
    user_id: str = "agent",
    project_id: str = "",
) -> str:
    """Manage A/B experiments — create, run, check status, view results, or list experiments.

    Args:
        action: One of "create", "run", "status", "results", "list".
        experiment_id: Experiment ID (for run/status/results).
        name: Experiment name (for create).
        dataset_name: Langfuse dataset name (for create).
        variants: JSON array of variant configs, e.g. [{"label": "v1", "model": "openai:gpt-4o", "prompt": "..."}].
        max_items: Maximum dataset items to use (for create, default 50).
        auto_score: Whether to auto-score outputs (default True).
        user_id: User ID for tracking.
        project_id: Optional project ID.
    """
    profile = _get_profile()
    resolved_uid = _resolve_user_id(user_id)

    if action == "list":
        items, total = list_experiments(resolved_uid, limit=50)
        return json.dumps({"items": items, "total": total}, ensure_ascii=False, indent=2)

    if action == "create":
        if not name:
            return _err("name is required for create")
        variant_list = _json_safe_list(variants)
        config = {
            "dataset_name": dataset_name,
            "variants": variant_list,
            "max_items": max_items,
            "auto_score": auto_score,
        }
        exp_id = create_experiment(
            user_id=resolved_uid,
            name=name,
            experiment_type="ab_test",
            config=config,
            description=f"A/B test on dataset: {dataset_name}",
            project_id=project_id or None,
        )
        return json.dumps({"ok": True, "experiment_id": exp_id, "action": "created"}, ensure_ascii=False)

    if action == "status":
        exp = get_experiment(experiment_id)
        if not exp:
            return _err(f"Experiment {experiment_id} not found")
        runs, total_runs = list_experiment_runs(experiment_id, limit=500)
        status_counts: dict[str, int] = {}
        for r in runs:
            s = r.get("status", "unknown")
            status_counts[s] = status_counts.get(s, 0) + 1
        return json.dumps({
            "experiment": exp,
            "run_counts": status_counts,
            "total_runs": total_runs,
        }, ensure_ascii=False, indent=2)

    if action == "results":
        exp = get_experiment(experiment_id)
        if not exp:
            return _err(f"Experiment {experiment_id} not found")
        runs, _total = list_experiment_runs(experiment_id, limit=500)

        # Aggregate per variant
        variant_stats: dict[str, dict[str, Any]] = {}
        for r in runs:
            label = r.get("variant_label", "unknown")
            if label not in variant_stats:
                variant_stats[label] = {"scores_sum": {}, "count": 0, "latencies": [], "model": r.get("model_spec")}
            variant_stats[label]["count"] += 1
            scores = r.get("scores", {})
            if isinstance(scores, dict) and "parse_error" not in scores:
                for k, v in scores.items():
                    try:
                        variant_stats[label]["scores_sum"][k] = variant_stats[label]["scores_sum"].get(k, 0) + float(v)
                    except (ValueError, TypeError):
                        pass
            lat = r.get("latency_ms")
            if lat is not None:
                variant_stats[label]["latencies"].append(float(lat))

        summary = {}
        for label, stats in variant_stats.items():
            cnt = stats["count"]
            avg_scores = {k: round(v / cnt, 3) for k, v in stats["scores_sum"].items()} if cnt > 0 else {}
            avg_latency = round(mean(stats["latencies"]), 2) if stats["latencies"] else None
            summary[label] = {
                "model": stats["model"],
                "run_count": cnt,
                "avg_scores": avg_scores,
                "avg_latency_ms": avg_latency,
            }

        return json.dumps({
            "experiment_id": experiment_id,
            "name": exp.get("name"),
            "status": exp.get("status"),
            "variant_summary": summary,
        }, ensure_ascii=False, indent=2)

    if action == "run":
        exp = get_experiment(experiment_id)
        if not exp:
            return _err(f"Experiment {experiment_id} not found")
        config = exp.get("config", {})
        variant_list = config.get("variants", [])
        dataset_name_cfg = config.get("dataset_name", "")
        max_items_cfg = config.get("max_items", 50)
        auto_score_cfg = config.get("auto_score", True)

        if not variant_list:
            return _err("No variants configured")

        client, _cfg, err = _build_client_safe()
        if err:
            return _err(f"Langfuse connection failed: {err}")

        try:
            dataset = client.api.datasets.get(dataset_name_cfg)
            items_raw = dataset.items if hasattr(dataset, "items") else []
            dataset_items = list(items_raw)[:max_items_cfg]
        except Exception as e:
            return _err(f"Failed to fetch dataset '{dataset_name_cfg}': {e}")

        if not dataset_items:
            return _err(f"Dataset '{dataset_name_cfg}' has no items")

        try:
            update_experiment_status(experiment_id, "running")

            completed = 0
            failed = 0

            # Build work items for parallel execution
            work_items: list[tuple[str, str, dict, str | None]] = []
            for item in dataset_items:
                item_dict = _to_dict(item)
                input_str = _extract_text(item_dict.get("input"))
                for variant in variant_list:
                    work_items.append((
                        str(variant.get("label", "unknown")),
                        str(variant.get("model", "")),
                        variant,
                        input_str,
                    ))

            def _run_variant(args: tuple) -> tuple[bool, str]:
                label, model_raw, variant, input_str = args
                prompt_text = variant.get("prompt")
                try:
                    spec = _parse_model_spec(model_raw)
                    trace_stub: dict[str, Any] = {"input": input_str, "metadata": {}}
                    if prompt_text:
                        trace_stub["metadata"]["system_prompt"] = prompt_text

                    replay_result = _replay_trace(profile, trace_stub, spec, prompt_override=prompt_text)

                    run_id = create_experiment_run(
                        experiment_id=experiment_id,
                        variant_label=label,
                        model_spec=model_raw,
                        input_text=input_str,
                        prompt_text=prompt_text,
                    )

                    scores: dict = {}
                    if auto_score_cfg and replay_result.get("output"):
                        judge = _default_judge_spec(profile)
                        scores = _auto_score_response(profile, judge, input_str, str(replay_result["output"]))

                    update_experiment_run(
                        run_id,
                        output_text=str(replay_result.get("output", "")),
                        scores_json=scores,
                        latency_ms=replay_result.get("latency_ms"),
                        status="completed",
                    )
                    return True, label
                except Exception as e:
                    _log.warning("Experiment run failed [%s/%s]: %s", label, model_raw, e)
                    return False, f"{label}: {e}"

            max_workers = min(len(work_items), 8)
            with ThreadPoolExecutor(max_workers=max_workers) as executor:
                futures = [executor.submit(_run_variant, wi) for wi in work_items]
                for future in as_completed(futures):
                    ok, _info = future.result()
                    if ok:
                        completed += 1
                    else:
                        failed += 1

            status = "completed" if failed == 0 else ("failed" if completed == 0 else "completed")
            update_experiment_status(experiment_id, status)
            return json.dumps({
                "ok": True,
                "experiment_id": experiment_id,
                "completed_runs": completed,
                "failed_runs": failed,
            }, ensure_ascii=False)

        except Exception as e:
            _log.exception("experiment_manager run failed for exp %s", experiment_id)
            update_experiment_status(experiment_id, "failed", result_json={"error": str(e)})
            return _err(f"Experiment run failed: {e}")

    return _err(f"Unknown action: {action}")


# ---------------------------------------------------------------------------
# Tool 3: Langfuse Evaluation Pipeline
# ---------------------------------------------------------------------------

@_lc_tool("langfuse_evaluation_pipeline")
def langfuse_eval_tool(
    action: str,
    trace_ids: str = "[]",
    criteria: str = "[]",
    judge_model: str = "",
    filter_params: str = "{}",
    user_id: str = "agent",
    project_id: str = "",
) -> str:
    """Run LLM-as-Judge evaluation on Langfuse traces and store scores.
    Combines Langfuse native evaluation features with extended LLM-as-Judge scoring.

    Args:
        action: One of "run_eval", "list_scores", "score_summary".
        trace_ids: JSON array of trace IDs to evaluate.
        criteria: JSON array of criteria names (default: all 5 — relevance, coherence, helpfulness, accuracy, conciseness).
        judge_model: Judge model spec (e.g. "openai:gpt-4o"). Uses default if empty.
        filter_params: JSON object for filtering scores.
        user_id: User ID for tracking.
        project_id: Optional project ID.
    """
    profile = _get_profile()
    resolved_uid = _resolve_user_id(user_id)
    trace_id_list = _json_safe_list(trace_ids)
    criteria_list = _json_safe_list(criteria) or [
        "relevance", "coherence", "helpfulness", "accuracy", "conciseness",
    ]

    if action == "run_eval":
        if not trace_id_list:
            return _err("trace_ids required for run_eval")

        client, _cfg, err = _build_client_safe()
        if err:
            return _err(f"Langfuse connection failed: {err}")

        judge_spec = _parse_model_spec(judge_model) if judge_model else _default_judge_spec(profile)

        exp_id = create_experiment(
            user_id=resolved_uid,
            name=f"eval_{len(trace_id_list)}_traces",
            experiment_type="eval",
            config={"trace_ids": trace_id_list, "criteria": criteria_list, "judge_model": judge_model},
            project_id=project_id or None,
        )

        try:
            update_experiment_status(exp_id, "running")
            results: list[dict] = []

            def _eval_trace(tid: str) -> dict:
                try:
                    trace = _fetch_trace_detail(client, tid)
                    if not trace:
                        return {"trace_id": tid, "error": "not found"}
                    trace_data = _to_dict(trace)

                    input_str = _extract_text(trace_data.get("input"))
                    output_str = _extract_text(trace_data.get("output"))

                    if not output_str:
                        return {"trace_id": tid, "error": "no output in trace"}

                    scores = _auto_score_response(profile, judge_spec, input_str, output_str)

                    if "parse_error" not in scores:
                        scores = {k: v for k, v in scores.items() if k in criteria_list}

                    # Store scores in Langfuse
                    for score_name, score_value in scores.items():
                        if score_name == "parse_error":
                            continue
                        try:
                            client.score(
                                trace_id=tid,
                                name=f"llm_judge_{score_name}",
                                value=float(score_value),
                                comment=f"Auto-evaluated by LLM-as-Judge ({judge_spec.get('model', 'unknown')})",
                            )
                        except Exception:
                            pass

                    run_id = create_experiment_run(
                        experiment_id=exp_id,
                        variant_label="original",
                        model_spec="original",
                        input_text=input_str,
                        trace_id=tid,
                    )
                    update_experiment_run(
                        run_id,
                        output_text=output_str[:1000],
                        scores_json=scores,
                        status="completed",
                    )

                    return {"trace_id": tid, "scores": scores}
                except Exception as e:
                    _log.warning("Eval failed for trace %s: %s", tid, e)
                    return {"trace_id": tid, "error": str(e)}

            max_workers = min(len(trace_id_list), 5)
            with ThreadPoolExecutor(max_workers=max_workers) as executor:
                futures = {executor.submit(_eval_trace, tid): tid for tid in trace_id_list}
                for future in as_completed(futures):
                    results.append(future.result())

            update_experiment_status(exp_id, "completed", result_json=results)

            return json.dumps({
                "experiment_id": exp_id,
                "evaluated_count": len([r for r in results if "scores" in r]),
                "error_count": len([r for r in results if "error" in r]),
                "results": results,
            }, ensure_ascii=False, indent=2)

        except Exception as e:
            _log.exception("langfuse_eval_tool failed for exp %s", exp_id)
            update_experiment_status(exp_id, "failed", result_json={"error": str(e)})
            return _err(f"Evaluation failed: {e}")

    if action == "list_scores":
        client, _cfg, err = _build_client_safe()
        if err:
            return _err(f"Langfuse connection failed: {err}")
        params = _json_safe_dict(filter_params)
        try:
            scores_resp = client.api.score.list(**params)
            scores_data = _to_dict(scores_resp)
            return json.dumps(scores_data, ensure_ascii=False, default=str, indent=2)
        except Exception as e:
            return _err(f"Failed to list scores: {e}")

    if action == "score_summary":
        if not trace_id_list:
            return _err("trace_ids required for score_summary")

        client, _cfg, err = _build_client_safe()
        if err:
            return _err(f"Langfuse connection failed: {err}")

        all_scores: dict[str, list[float]] = {}
        for tid in trace_id_list:
            try:
                scores_resp = client.api.score.list(trace_id=tid)
                items = getattr(scores_resp, "data", []) or []
                for s in items:
                    s_dict = _to_dict(s)
                    sname = str(s_dict.get("name", ""))
                    val = s_dict.get("value")
                    if sname and val is not None:
                        try:
                            all_scores.setdefault(sname, []).append(float(val))
                        except (ValueError, TypeError):
                            pass
            except Exception:
                pass

        summary = {}
        for sname, values in all_scores.items():
            summary[sname] = {
                "count": len(values),
                "avg": round(mean(values), 3) if values else 0,
                "min": round(min(values), 3) if values else 0,
                "max": round(max(values), 3) if values else 0,
            }

        return json.dumps({
            "trace_count": len(trace_id_list),
            "score_summary": summary,
        }, ensure_ascii=False, indent=2)

    return _err(f"Unknown action: {action}")


# ---------------------------------------------------------------------------
# Tool 4: Response Comparison & Ranking
# ---------------------------------------------------------------------------

@_lc_tool("response_comparison_ranking")
def response_comparison_tool(
    trace_ids: str = "[]",
    model_specs: str = "[]",
    ranking_model: str = "",
    user_id: str = "agent",
    project_id: str = "",
) -> str:
    """Compare multiple model responses for given traces and produce a ranking.
    Fetches original trace outputs, optionally replays with additional models,
    then uses an LLM ranker to compare and rank all responses.

    Args:
        trace_ids: JSON array of trace IDs to compare.
        model_specs: JSON array of additional model specs to replay with (optional).
        ranking_model: Model to use for ranking (e.g. "openai:gpt-4o"). Uses default if empty.
        user_id: User ID for tracking.
        project_id: Optional project ID.
    """
    profile = _get_profile()
    resolved_uid = _resolve_user_id(user_id)
    tid_list = _json_safe_list(trace_ids)
    extra_specs_raw = _json_safe_list(model_specs)

    if not tid_list:
        return _err("trace_ids required")

    client, _cfg, err = _build_client_safe()
    if err:
        return _err(f"Langfuse connection failed: {err}")

    ranker_spec = _parse_model_spec(ranking_model) if ranking_model else _default_judge_spec(profile)

    extra_specs = []
    for raw in extra_specs_raw:
        try:
            extra_specs.append(_parse_model_spec(raw))
        except Exception:
            pass

    exp_id = create_experiment(
        user_id=resolved_uid,
        name=f"comparison_{len(tid_list)}_traces",
        experiment_type="comparison",
        config={"trace_ids": tid_list, "model_specs": extra_specs_raw, "ranking_model": ranking_model},
        project_id=project_id or None,
    )

    try:
        update_experiment_status(exp_id, "running")
        all_rankings: list[dict] = []

        def _compare_trace(tid: str) -> dict:
            try:
                trace = _fetch_trace_detail(client, tid)
                if not trace:
                    return {"trace_id": tid, "error": "not found"}
                trace_data = _to_dict(trace)

                input_str = _extract_text(trace_data.get("input"))
                outp = _extract_text(trace_data.get("output"))
                responses = {"original": outp}

                # Replay with extra models (parallel within trace)
                def _replay_spec(spec: dict) -> tuple[str, str, dict | None]:
                    label = str(spec.get("alias", spec.get("model", "unknown")))
                    try:
                        result = _replay_trace(profile, trace_data, spec)
                        return label, str(result.get("output", "")), result
                    except Exception as e:
                        return label, f"(error: {e})", None

                if extra_specs:
                    with ThreadPoolExecutor(max_workers=min(len(extra_specs), 5)) as pool:
                        spec_futures = {pool.submit(_replay_spec, sp): sp for sp in extra_specs}
                        for fut in as_completed(spec_futures):
                            label, output, result = fut.result()
                            responses[label] = output
                            if result:
                                sp = spec_futures[fut]
                                run_id = create_experiment_run(
                                    experiment_id=exp_id,
                                    variant_label=label,
                                    model_spec=f"{sp.get('provider', '')}:{sp.get('model', '')}",
                                    input_text=input_str,
                                    trace_id=tid,
                                )
                                update_experiment_run(
                                    run_id,
                                    output_text=output[:1000],
                                    latency_ms=result.get("latency_ms"),
                                    status="completed",
                                )

                # Save original as run
                run_id = create_experiment_run(
                    experiment_id=exp_id,
                    variant_label="original",
                    model_spec="original",
                    input_text=input_str,
                    trace_id=tid,
                )
                update_experiment_run(run_id, output_text=outp[:1000], status="completed")

                # Rank responses
                if len(responses) >= 2:
                    ranking = _compare_responses_with_llm(profile, ranker_spec, input_str, responses)
                else:
                    ranking = {"ranking": [{"variant": "original", "rank": 1, "reasoning": "Only one response"}], "best": "original"}

                # Store ranking scores in Langfuse
                try:
                    for item in ranking.get("ranking", []):
                        variant = item.get("variant", "")
                        rank = item.get("rank", 0)
                        if variant and rank:
                            client.score(
                                trace_id=tid,
                                name=f"comparison_rank_{variant}",
                                value=float(1.0 / rank),
                                comment=item.get("reasoning", ""),
                            )
                except Exception:
                    pass

                return {
                    "trace_id": tid,
                    "input": input_str[:200],
                    "ranking": ranking,
                }
            except Exception as e:
                _log.warning("Comparison failed for trace %s: %s", tid, e)
                return {"trace_id": tid, "error": str(e)}

        # Process traces in parallel
        max_workers = min(len(tid_list), 5)
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            futures = {executor.submit(_compare_trace, tid): tid for tid in tid_list}
            for future in as_completed(futures):
                all_rankings.append(future.result())

        update_experiment_status(exp_id, "completed", result_json=all_rankings)

        return json.dumps({
            "experiment_id": exp_id,
            "comparison_count": len(tid_list),
            "rankings": all_rankings,
        }, ensure_ascii=False, indent=2)

    except Exception as e:
        _log.exception("response_comparison_tool failed for exp %s", exp_id)
        update_experiment_status(exp_id, "failed", result_json={"error": str(e)})
        return _err(f"Comparison failed: {e}")
