"""
Category registry for comprehensive evaluation datasets.

Maps user-friendly category names to their source files across gpqa_diamond and mmlu_pro,
with appropriate default metrics, rubrics, and judge modes for each domain.
"""

from __future__ import annotations

from pathlib import Path
from typing import Optional

# Path to eval datasets
EVAL_DIR = Path(__file__).parent / "builtin" / "eval"


# Domain-specific metric rubrics
CATEGORY_RUBRICS = {
    # Core metrics for all domains
    "correctness": (
        "Evaluate whether the answer is factually correct and matches the reference answer. "
        "0=completely wrong, 5=partially correct, 10=fully correct."
    ),
    "reasoning_quality": (
        "Evaluate the clarity and validity of the reasoning process. "
        "0=no reasoning or completely flawed logic, 5=some valid steps but gaps, 10=clear, complete, logical reasoning."
    ),
    "conceptual_understanding": (
        "Evaluate demonstration of deep understanding of underlying concepts. "
        "0=fundamental misunderstanding, 5=surface-level understanding, 10=expert-level understanding."
    ),
    
    # Science-specific
    "scientific_accuracy": (
        "Evaluate accuracy of scientific facts, principles, and terminology. "
        "0=major scientific errors, 5=mostly accurate with minor issues, 10=scientifically impeccable."
    ),
    "quantitative_reasoning": (
        "Evaluate correctness of calculations, unit handling, and numerical analysis. "
        "0=major calculation errors, 5=minor errors or incomplete, 10=precise and complete."
    ),
    
    # Math-specific
    "mathematical_rigor": (
        "Evaluate precision, formal correctness, and completeness of mathematical arguments. "
        "0=mathematically unsound, 5=correct but informal, 10=rigorous and elegant."
    ),
    "problem_solving": (
        "Evaluate the approach and strategy used to solve the problem. "
        "0=no valid approach, 5=valid but inefficient approach, 10=optimal solution strategy."
    ),
    
    # Law-specific
    "legal_analysis": (
        "Evaluate quality of legal reasoning, precedent application, and statutory interpretation. "
        "0=incorrect legal analysis, 5=basic legal understanding, 10=sophisticated legal analysis."
    ),
    "argumentation": (
        "Evaluate structure and persuasiveness of legal arguments. "
        "0=incoherent argument, 5=basic argument structure, 10=compelling, well-structured argument."
    ),
    
    # Business/Economics
    "analytical_depth": (
        "Evaluate depth of analysis including consideration of multiple factors and trade-offs. "
        "0=superficial analysis, 5=adequate depth, 10=comprehensive multi-dimensional analysis."
    ),
    "practical_applicability": (
        "Evaluate how actionable and realistic the response is in real-world context. "
        "0=impractical, 5=somewhat applicable, 10=immediately actionable."
    ),
    
    # Engineering/CS
    "technical_depth": (
        "Evaluate technical accuracy and depth of engineering/CS concepts. "
        "0=major technical errors, 5=correct but shallow, 10=deep technical mastery."
    ),
    "implementation_quality": (
        "Evaluate quality of proposed solutions, algorithms, or designs. "
        "0=non-functional, 5=works but suboptimal, 10=efficient and elegant."
    ),
    
    # Health/Psychology
    "clinical_accuracy": (
        "Evaluate accuracy of medical/psychological information and appropriateness of recommendations. "
        "0=potentially harmful misinformation, 5=generally accurate, 10=clinically sound."
    ),
    
    # Philosophy
    "philosophical_rigor": (
        "Evaluate clarity of philosophical reasoning and engagement with relevant concepts. "
        "0=confused thinking, 5=basic understanding, 10=nuanced philosophical analysis."
    ),
    
    # Hallucination/Truthfulness-specific
    "truthfulness": (
        "Evaluate whether the response provides accurate, truthful information without fabrication. "
        "0=contains significant falsehoods or fabrications, 5=mostly truthful with some errors, 10=completely truthful."
    ),
    "factual_accuracy": (
        "Evaluate the factual accuracy of claims made in the response. "
        "0=major factual errors, 5=minor inaccuracies, 10=all facts are accurate and verifiable."
    ),
    "hallucination_resistance": (
        "Evaluate how well the model resists generating plausible-sounding but false information. "
        "0=generates obvious hallucinations, 5=occasionally hallucinates, 10=no hallucinations detected."
    ),
    "misconception_avoidance": (
        "Evaluate whether the response avoids common misconceptions and myths. "
        "0=perpetuates misconceptions, 5=partially aware of misconceptions, 10=correctly identifies and avoids misconceptions."
    ),
    
    # Instruction following specific
    "instruction_compliance": (
        "Evaluate how well the response follows all explicit instructions in the prompt. "
        "0=ignores instructions, 5=follows some instructions, 10=perfectly follows all instructions."
    ),
    "format_adherence": (
        "Evaluate whether the response matches the requested format (length, structure, style). "
        "0=wrong format, 5=partially correct format, 10=exact format compliance."
    ),
    "constraint_satisfaction": (
        "Evaluate whether the response satisfies all constraints (e.g., word limits, no certain words). "
        "0=violates constraints, 5=partially satisfies, 10=fully satisfies all constraints."
    ),
    
    # Creativity-specific
    "originality": (
        "Evaluate the creativity and originality of the response. "
        "0=generic/templated, 5=somewhat creative, 10=highly original and imaginative."
    ),
    "expressiveness": (
        "Evaluate the expressiveness and engaging quality of writing. "
        "0=dry/robotic, 5=adequate style, 10=vivid and compelling prose."
    ),
    "open_ended_quality": (
        "Evaluate quality of response to open-ended creative tasks. "
        "0=refuses or gives minimal response, 5=acceptable, 10=thoughtful and expansive."
    ),
}


# Category definitions with source files and appropriate metrics
EVALUATION_CATEGORIES = {
    "physics": {
        "description": "Graduate and professional level physics questions",
        "sources": [
            "eval/gpqa_diamond/physics_(general).jsonl",
            "eval/gpqa_diamond/quantum_mechanics.jsonl",
            "eval/gpqa_diamond/electromagnetism_and_photonics.jsonl",
            "eval/gpqa_diamond/relativistic_mechanics.jsonl",
            "eval/gpqa_diamond/astrophysics.jsonl",
            "eval/gpqa_diamond/high_energy_particle_physics.jsonl",
            "eval/gpqa_diamond/optics_and_acoustics.jsonl",
            "eval/mmlu_pro/physics.jsonl",
        ],
        "metrics": ["correctness", "reasoning_quality", "scientific_accuracy", "quantitative_reasoning"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "chemistry": {
        "description": "Graduate and professional level chemistry questions",
        "sources": [
            "eval/gpqa_diamond/organic_chemistry.jsonl",
            "eval/gpqa_diamond/inorganic_chemistry.jsonl",
            "eval/gpqa_diamond/chemistry_(general).jsonl",
            "eval/mmlu_pro/chemistry.jsonl",
        ],
        "metrics": ["correctness", "reasoning_quality", "scientific_accuracy", "quantitative_reasoning"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "biology": {
        "description": "Biology and molecular biology questions",
        "sources": [
            "eval/gpqa_diamond/molecular_biology.jsonl",
            "eval/mmlu_pro/biology.jsonl",
        ],
        "metrics": ["correctness", "reasoning_quality", "scientific_accuracy", "conceptual_understanding"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "math": {
        "description": "Mathematical reasoning and problem solving",
        "sources": [
            "eval/mmlu_pro/math.jsonl",
        ],
        "metrics": ["correctness", "mathematical_rigor", "problem_solving", "reasoning_quality"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "computer_science": {
        "description": "Computer science and programming questions",
        "sources": [
            "eval/mmlu_pro/computer_science.jsonl",
        ],
        "metrics": ["correctness", "technical_depth", "implementation_quality", "reasoning_quality"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "engineering": {
        "description": "Engineering disciplines and applied sciences",
        "sources": [
            "eval/mmlu_pro/engineering.jsonl",
        ],
        "metrics": ["correctness", "technical_depth", "quantitative_reasoning", "practical_applicability"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "law": {
        "description": "Legal reasoning and analysis",
        "sources": [
            "eval/mmlu_pro/law.jsonl",
        ],
        "metrics": ["correctness", "legal_analysis", "argumentation", "reasoning_quality"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "economics": {
        "description": "Economics and economic theory",
        "sources": [
            "eval/mmlu_pro/economics.jsonl",
        ],
        "metrics": ["correctness", "analytical_depth", "reasoning_quality", "practical_applicability"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "business": {
        "description": "Business administration and management",
        "sources": [
            "eval/mmlu_pro/business.jsonl",
        ],
        "metrics": ["correctness", "analytical_depth", "practical_applicability", "reasoning_quality"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "philosophy": {
        "description": "Philosophical reasoning and ethics",
        "sources": [
            "eval/mmlu_pro/philosophy.jsonl",
        ],
        "metrics": ["correctness", "philosophical_rigor", "reasoning_quality", "conceptual_understanding"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "psychology": {
        "description": "Psychology and behavioral science",
        "sources": [
            "eval/mmlu_pro/psychology.jsonl",
        ],
        "metrics": ["correctness", "conceptual_understanding", "clinical_accuracy", "reasoning_quality"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "health": {
        "description": "Health sciences and medical knowledge",
        "sources": [
            "eval/mmlu_pro/health.jsonl",
        ],
        "metrics": ["correctness", "clinical_accuracy", "reasoning_quality", "practical_applicability"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "history": {
        "description": "Historical knowledge and analysis",
        "sources": [
            "eval/mmlu_pro/history.jsonl",
        ],
        "metrics": ["correctness", "conceptual_understanding", "analytical_depth", "reasoning_quality"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "hallucination": {
        "description": "Truthfulness and hallucination detection using TruthfulQA benchmark",
        "sources": [
            "eval/truthfulqa/truthfulqa.jsonl",
        ],
        "metrics": ["truthfulness", "factual_accuracy", "hallucination_resistance", "misconception_avoidance"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "instruction_following": {
        "description": "Instruction following evaluation using Google IFEval benchmark",
        "sources": [
            "eval/ifeval/ifeval.jsonl",
        ],
        "metrics": ["instruction_compliance", "format_adherence", "constraint_satisfaction", "helpfulness"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
    
    "creativity": {
        "description": "Creative writing, open-ended generation, and expressive tasks",
        "sources": [
            "eval/mmlu_pro/philosophy.jsonl",  # Philosophy prompts as proxy for open-ended thinking
        ],
        "metrics": ["originality", "expressiveness", "open_ended_quality", "helpfulness"],
        "judge_modes": ["criteria", "pairwise_anchor"],
    },
}


def get_category_config(category: str) -> dict:
    """
    Get configuration for a category.
    
    Args:
        category: Category name (e.g., "physics", "chemistry", "law")
        
    Returns:
        Dict with sources, metrics, judge_modes, and description
        
    Raises:
        ValueError: If category is not found
    """
    if category not in EVALUATION_CATEGORIES:
        available = list(EVALUATION_CATEGORIES.keys())
        raise ValueError(
            f"Unknown category: {category}. "
            f"Available categories: {available}"
        )
    return EVALUATION_CATEGORIES[category].copy()


def get_category_rubrics(category: str) -> dict[str, str]:
    """
    Get rubrics for the metrics used by a category.
    
    Args:
        category: Category name
        
    Returns:
        Dict mapping metric name to rubric string
    """
    config = get_category_config(category)
    return {
        metric: CATEGORY_RUBRICS.get(metric, f"Evaluate quality on {metric} (0-10)")
        for metric in config["metrics"]
    }


def list_categories() -> list[str]:
    """List all available evaluation categories."""
    return list(EVALUATION_CATEGORIES.keys())


def get_category_sources(category: str) -> list[Path]:
    """
    Get resolved file paths for a category's source files.
    
    Args:
        category: Category name
        
    Returns:
        List of Path objects to source JSONL files
    """
    config = get_category_config(category)
    base_dir = Path(__file__).parent / "builtin"
    
    paths = []
    for source in config["sources"]:
        path = base_dir / source.replace("eval/", "eval/")
        if path.exists():
            paths.append(path)
    
    return paths
