from pathlib import Path
from typing import Any

from inspect_ai import eval
from inspect_ai.log import EvalLog

from bench_af._abstract.objects import Environment, ModelOrganism


def validate_model_organism(
    *,
    model_organism: ModelOrganism,
    environment: Environment,
    raw_eval_logs: bool = True,
    eval_config: dict[str, Any],
    results_dir: Path,
    head: int = 50,
) -> list[EvalLog] | str:
    """
    Validate a model organism on an environment.
    """

    # Check if environment is supported by model organism
    if environment.name not in model_organism.supported_environments:
        raise ValueError(
            f"Environment {environment.name} not supported by model organism {model_organism.name}"
        )

    # Get the observed task from the environment for validation
    observed_task = environment.get_observed_task(head=head)
    unobserved_task = environment.get_unobserved_task(head=head)
    tools = environment.get_tools()
    model_organism.update_solver(tools=tools)

    eval_logs = eval(
        tasks=[observed_task, unobserved_task],
        model=model_organism.model,
        solver=model_organism.solver,
        tags=["validate_model_organism"],
        log_dir=str(results_dir / "logs"),
        log_format="json",
        **eval_config,
    )

    if raw_eval_logs:
        return eval_logs
    else:
        return _parse_eval_logs(eval_logs)


def _parse_eval_log(eval_log: EvalLog) -> str:
    """
    Parse the results of an eval log.
    """
    if eval_log.status == "success":
        # TODO: Fix this usage of a deprecated attribute results.metrics
        if eval_log.results is None:
            raise ValueError("Eval log results are None")
        return f"Accuracy: {eval_log.results.metrics['accuracy'].value}"
    else:
        return f"Error: {eval_log.error}"


def _parse_eval_logs(eval_logs: list[EvalLog]) -> str:
    """
    Parse the results of a list of eval logs.
    """
    return "\n".join(_parse_eval_log(eval_log) for eval_log in eval_logs)
