from __future__ import annotations

from typing import Any

DEFAULT_EVAL_NAME = "edge_side_pane_comprehensiveness_freechat"
METRIC_OPT_DIRECTIONS = {"higher_better", "lower_better"}


def _metric(
    name: str,
    *,
    is_llm_judged: bool = True,
    direction: str = "higher_better",
    description: str = "",
    purpose: str = "",
) -> dict[str, Any]:
    """Build a metric definition dict.

    For non-LLM-judged (rule-based) metrics, ``description`` and ``purpose``
    are included so downstream consumers know what the metric measures.
    For LLM-judged metrics these fields are omitted (empty strings ignored).
    """
    entry: dict[str, Any] = {
        "name": name,
        "is_llm_judged": is_llm_judged,
        "optimization_direction": direction,
    }
    if description:
        entry["description"] = description
    if purpose:
        entry["measurement_purpose"] = purpose
    return entry


# Shared comprehensiveness metrics (edge side-pane picasso evals)
_COMPREHENSIVENESS_METRICS: list[dict[str, Any]] = [
    _metric("completeness"),
    _metric("clarity & readability"),
    _metric("engagement"),
    _metric("factual accuracy"),
    _metric("groundedness"),
]


# Dataset -> eval_name/key_metrics mapping shared by API/SDK/CLI.
# Keep this aligned with ui-core/src/lib/constants.ts.
#
# Each metric in key_metrics is a dict with:
#   name                  - metric score name
#   is_llm_judged         - True if scored by an LLM judge
#   optimization_direction- "higher_better" or "lower_better"
#   description           - (non-LLM only) what the metric measures
#   measurement_purpose   - (non-LLM only) why the metric exists
DATASET_TO_EVAL_MAP: dict[str, dict[str, Any]] = {
    # ---- Edge side-pane comprehensiveness evals (picasso) ----
    "picasso_cn_page_summary_0610": {
        "eval_name": "edge_side_pane_comprehensiveness_pagesummary",
        "key_metrics": _COMPREHENSIVENESS_METRICS,
    },
    "picasso_cn_expand_topic_1113": {
        "eval_name": "edge_side_pane_comprehensiveness_expandtopic",
        "key_metrics": _COMPREHENSIVENESS_METRICS,
    },
    "picasso_cn_chat_0923": {
        "eval_name": "edge_side_pane_comprehensiveness_freechat",
        "key_metrics": _COMPREHENSIVENESS_METRICS,
    },
    "picasso_cn_expand_topic_0722": {
        "eval_name": "edge_side_pane_comprehensiveness_single_tab",
        "key_metrics": _COMPREHENSIVENESS_METRICS,
    },
    # ---- Think-deeper response quality ----
    "td_response_quality": {
        "eval_name": "td_response_quality",
        "key_metrics": [
            _metric("td_response_quality_accuracy"),
            _metric("td_response_quality_completeness"),
            _metric("td_response_quality_depth_thoughtfulness"),
            _metric("td_response_quality_logical_soundness"),
            _metric("td_response_quality_overall_satisfaction"),
            _metric("td_response_quality_readability_clarity"),
            _metric("td_response_quality_relevance"),
        ],
    },
    # ---- Leaderboard evals from aipa-evals ----
    "tool_hallucination_payload": {
        "eval_name": "model_hallucination",
        "key_metrics": [
            _metric("Hallucination", is_llm_judged=False, direction="lower_better", description="Detects if response contains hallucinated tool/action names or query patterns", purpose="Ensure model doesn't output internal tool names or JSON structures that should only appear in backend processing"),
            _metric("EmptyResponse", is_llm_judged=False, direction="lower_better", description="Checks if response is empty or whitespace-only", purpose="Detect cases where model fails to generate any meaningful output"),
            _metric("OutputLength", is_llm_judged=False, direction="lower_better", description="Normalized response length (length/100000)", purpose="Track response verbosity as a general metric"),
            _metric("any_search_tool_trigger_rate", is_llm_judged=False, direction="higher_better", description="Checks if any search tool was triggered", purpose="Track search tool usage patterns across hallucination test scenarios"),
        ],
    },
    # ---- Safety evals ----
    "dea": {
        "eval_name": "dea_v2",
        "key_metrics": [
            _metric("safety_eval_dea_disparaging"),
            _metric("safety_eval_dea_existential"),
            _metric("safety_eval_dea_argumentative"),
        ],
    },
    "sexual_v2": {
        "eval_name": "safety_sexual_v2",
        "key_metrics": [_metric("safety_eval_sexualrt_sexual")],
    },
    "violence_v2": {
        "eval_name": "safety_violence_v2",
        "key_metrics": [_metric("safety_eval_violencert_violence")],
    },
    "hatefairness_v2": {
        "eval_name": "safety_hatefairness_v2",
        "key_metrics": [_metric("safety_eval_hatefairnessrt_hate_and_fairness")],
    },
    "selfharm_v2": {
        "eval_name": "safety_selfharm_v2",
        "key_metrics": [_metric("safety_eval_selfharmrt_self_harm")],
    },
    "jailbreak": {
        "eval_name": "jailbreak_v2",
        "key_metrics": [_metric("safety_eval_jailbreak_jailbreak")],
    },
    "romance": {
        "eval_name": "romance",
        "key_metrics": [_metric("safety_eval_romance_romance")],
    },
    "mentalhealth": {
        "eval_name": "mentalhealth",
        "key_metrics": [_metric("safety_eval_mentalhealth_mental_health")],
    },
    "elections": {
        "eval_name": "elections",
        "key_metrics": [_metric("safety_eval_elections_elections")],
    },
    "politics": {
        "eval_name": "politics",
        "key_metrics": [_metric("safety_eval_politics_politics")],
    },
    "hui": {
        "eval_name": "harmful_unethical_illegal",
        "key_metrics": [_metric("safety_eval_hui_hui")],
    },
    # ---- Groundedness ----
    "grounding": {
        "eval_name": "groundedness_v2",
        "key_metrics": [
            _metric("groundedness"),
            _metric("groundedness_all_sources"),
            _metric("groundedness_judge_tips"),
        ],
    },
    # ---- Verbosity ----
    "verbosity": {
        "eval_name": "verbosity",
        "key_metrics": [_metric("not_verbose")],
    },
    # ---- Token efficiency ----
    "multiple_tools_750": {
        "eval_name": "token_efficiency",
        "key_metrics": [
            _metric("prompt_tokens_count", is_llm_judged=False, direction="lower_better", description="Total prompt tokens across all inferences, normalized by 100k", purpose="Track input token usage for cost/efficiency optimization"),
            _metric("prompt_cached_tokens_count", is_llm_judged=False, description="Total cached prompt tokens across all inferences, normalized by 100k", purpose="Track cache effectiveness for prompt caching optimization"),
            _metric("completion_tokens_count", is_llm_judged=False, direction="lower_better", description="Total completion tokens across all inferences, normalized by 100k", purpose="Track output token usage for cost/efficiency optimization"),
            _metric("prompt_cache_hit_rate", is_llm_judged=False, description="Ratio of cached tokens to total prompt tokens (0-1)", purpose="Measure effectiveness of prompt caching"),
        ],
    },
    # ---- Citations ----
    "groundedness": {
        "eval_name": "citations",
        "key_metrics": [
            _metric("has_citations", is_llm_judged=False, description="Binary check if response contains any citations", purpose="Ensure model provides source attribution when needed"),
            _metric("citation_count", is_llm_judged=False, description="Number of citations in the response (normalized)", purpose="Track citation quantity"),
            _metric("picasso_response_reference_section", is_llm_judged=False, description="Has reference section when citations exist", purpose="Detect if model adds explicit reference section formatting"),
            _metric("good_citations"),
            _metric("good_citations_on_cited"),
        ],
    },
    # ---- Instruction following ----
    "instruction_following": {
        "eval_name": "instruction_following",
        "key_metrics": [_metric("percentage_of_instructions_followed")],
    },
    # ---- Refusal ----
    "refusal_v2": {
        "eval_name": "refusal_v2",
        "key_metrics": [
            _metric("CR1"),
            _metric("CR2"),
            _metric("CR3"),
            _metric("CR4"),
            _metric("IR1", direction="lower_better"),
            _metric("IR2", direction="lower_better"),
            _metric("IR3", direction="lower_better"),
            _metric("IR4", direction="lower_better"),
            _metric("MT1", direction="lower_better"),
            _metric("NR1"),
            _metric("OT1"),
            _metric("align_with_ssr"),
        ],
    },
    # ---- Overpromising ----
    "overpromising": {
        "eval_name": "overpromising",
        "key_metrics": [
            _metric("not_overpromising"),
            _metric("overpromising_action_classifier_score"),
        ],
    },
    # ---- Completeness ----
    "completeness_v2": {
        "eval_name": "completeness_v2",
        "key_metrics": [_metric("completeness")],
    },
    # ---- Personality ----
    "AIPA Personality 250": {
        "eval_name": "personality_evals",
        "key_metrics": [_metric("personality_score_standard")],
    },
    # ---- Formatting ----
    "formatting": {
        "eval_name": "formatting_v2",
        "key_metrics": [_metric("formatting_v2_rating")],
    },
    # ---- Hello world (verbosity thresholds) ----
    "hello_world": {
        "eval_name": "hello_world",
        "key_metrics": [
            _metric("verbosity_eval_50", is_llm_judged=False, description="Verbosity threshold check at 50 tokens", purpose="Track response conciseness at 50-token limit"),
            _metric("verbosity_eval_100", is_llm_judged=False, description="Verbosity threshold check at 100 tokens", purpose="Track response conciseness at 100-token limit"),
            _metric("verbosity_eval_150", is_llm_judged=False, description="Verbosity threshold check at 150 tokens", purpose="Track response conciseness at 150-token limit"),
            _metric("verbosity_eval_200", is_llm_judged=False, description="Verbosity threshold check at 200 tokens", purpose="Track response conciseness at 200-token limit"),
            _metric("verbosity_eval_250", is_llm_judged=False, description="Verbosity threshold check at 250 tokens", purpose="Track response conciseness at 250-token limit"),
        ],
    },
    # ---- Backstory ----
    "updated_backstory_v2": {
        "eval_name": "backstory",
        "key_metrics": [_metric("BackstoryJudge")],
    },
    # ---- Factuality ----
    "quality": {
        "eval_name": "factuality",
        "key_metrics": [
            _metric("grounded_claims", is_llm_judged=False, description="Number of claims grounded by search results", purpose="Track factual grounding of response claims"),
            _metric("num_searchable_claims", is_llm_judged=False, description="Number of claims that can be verified via search", purpose="Track volume of verifiable claims in response"),
            _metric("factuality_rating"),
            _metric("severity_rating", direction="lower_better"),
        ],
    },
    # ---- IP / Copyright ----
    "ipcopyright": {
        "eval_name": "ipcopyright_v2",
        "key_metrics": [
            _metric("3gram_overlap", is_llm_judged=False, direction="lower_better", description="3-gram overlap ratio with copyrighted content", purpose="Detect potential copyright infringement via trigram matching"),
            _metric("5gram_overlap", is_llm_judged=False, direction="lower_better", description="5-gram overlap ratio with copyrighted content", purpose="Detect potential copyright infringement via 5-gram matching"),
            _metric("10gram_overlap", is_llm_judged=False, direction="lower_better", description="10-gram overlap ratio with copyrighted content", purpose="Detect potential copyright infringement via 10-gram matching"),
            _metric("20gram_overlap", is_llm_judged=False, direction="lower_better", description="20-gram overlap ratio with copyrighted content", purpose="Detect potential copyright infringement via 20-gram matching"),
            _metric("unacceptable_lcs_length", is_llm_judged=False, direction="lower_better", description="Length of unacceptable longest common subsequence with source", purpose="Detect long verbatim copies from copyrighted sources"),
            _metric("overall_defect_rate", is_llm_judged=False, direction="lower_better", description="Overall copyright defect rate across all checks", purpose="Aggregate copyright violation risk score"),
        ],
    },
}
