# This file was auto-generated by Fern from our API Definition.

import typing

import pydantic
from ..core.pydantic_utilities import IS_PYDANTIC_V2
from ..core.unchecked_base_model import UncheckedBaseModel
from .evaluation_config_input_target import EvaluationConfigInputTarget
from .test_case import TestCase


class EvaluationConfigInput(UncheckedBaseModel):
    """
    Evaluation configuration (reusable test suite specification).

    This represents the WHAT and HOW of an evaluation:
    - WHAT: The target being evaluated (workflow, agent, tool, etc.)
    - HOW: The test cases and execution settings

    This config can be:
    1. Saved as an Evaluation entity (reusable)
    2. Used directly for ephemeral runs (one-time)
    """

    target: EvaluationConfigInputTarget = pydantic.Field()
    """
    What to evaluate (workflow/agent/tool/prompt)
    """

    test_cases: typing.List[TestCase] = pydantic.Field()
    """
    Test cases to run
    """

    scorer_registry: typing.Optional[
        typing.Dict[str, typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]]
    ] = pydantic.Field(default=None)
    """
    Registry of scorer definitions used in test cases (scorer_id -> Scorer dict)
    """

    max_concurrent: typing.Optional[int] = pydantic.Field(default=None)
    """
    Maximum concurrent test executions
    """

    timeout_ms: typing.Optional[int] = pydantic.Field(default=None)
    """
    Timeout per test in milliseconds
    """

    stop_on_first_failure: typing.Optional[bool] = pydantic.Field(default=None)
    """
    Stop evaluation after first test failure
    """

    tags: typing.Optional[typing.List[str]] = pydantic.Field(default=None)
    """
    Tags for organization (model version, dataset, etc.)
    """

    metadata: typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]] = pydantic.Field(default=None)
    """
    Additional custom metadata
    """

    if IS_PYDANTIC_V2:
        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
    else:

        class Config:
            frozen = True
            smart_union = True
            extra = pydantic.Extra.allow
