# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.

from typing import List, Union, Optional
from datetime import datetime
from typing_extensions import Literal, TypeAlias

from pydantic import Field as FieldInfo

from .._models import BaseModel
from .eval_type import EvalType
from .application import Application

__all__ = [
    "EvalDefinition",
    "GlobalConfig",
    "GlobalConfigCriterion",
    "GlobalConfigGroundTruth",
    "GlobalConfigOutputMatchStringConfigOutput",
    "GlobalConfigOutputMatchStringConfigOutputExtract",
    "GlobalConfigOutputMatchListConfigOutput",
    "GlobalConfigOutputMatchListConfigOutputExtract",
]


class GlobalConfigCriterion(BaseModel):
    criterion: str
    """The criterion describes what our evaluation LLM must look for in the response.

    Remember that the answer to the criterion must be as a pass/fail.
    """


class GlobalConfigGroundTruth(BaseModel):
    ground_truth: str = FieldInfo(alias="groundTruth")
    """
    The ground truth is the most correct answer to the task that we measure the
    response against
    """


class GlobalConfigOutputMatchStringConfigOutputExtract(BaseModel):
    flags: str

    group: Union[int, List[int]]

    pattern: str


class GlobalConfigOutputMatchStringConfigOutput(BaseModel):
    type: Literal["string"]

    extract: Optional[GlobalConfigOutputMatchStringConfigOutputExtract] = None


class GlobalConfigOutputMatchListConfigOutputExtract(BaseModel):
    flags: str

    group: Union[int, List[int]]

    pattern: str


class GlobalConfigOutputMatchListConfigOutput(BaseModel):
    match_mode: Literal["exact_unordered", "contains"] = FieldInfo(alias="matchMode")

    type: Literal["list"]

    extract: Optional[GlobalConfigOutputMatchListConfigOutputExtract] = None

    pass_threshold: Optional[float] = FieldInfo(alias="passThreshold", default=None)

    score_metric: Optional[Literal["f1", "precision", "recall"]] = FieldInfo(alias="scoreMetric", default=None)


GlobalConfig: TypeAlias = Union[
    GlobalConfigCriterion,
    GlobalConfigGroundTruth,
    GlobalConfigOutputMatchStringConfigOutput,
    GlobalConfigOutputMatchListConfigOutput,
]


class EvalDefinition(BaseModel):
    id: str

    created_at: datetime = FieldInfo(alias="createdAt")
    """When the eval definition was created"""

    modified_at: datetime = FieldInfo(alias="modifiedAt")
    """When the eval definition was last modified"""

    name: str

    type: EvalType

    application: Optional[Application] = None
    """Application configuration and metadata"""

    global_config: Optional[GlobalConfig] = FieldInfo(alias="globalConfig", default=None)

    style_guide_id: Optional[str] = FieldInfo(alias="styleGuideId", default=None)
