# This file was auto-generated by Fern from our API Definition.

import typing

import pydantic
import typing_extensions
from ..core.pydantic_utilities import IS_PYDANTIC_V2
from ..core.serialization import FieldMetadata
from ..core.unchecked_base_model import UncheckedBaseModel


class EvaluationMetrics(UncheckedBaseModel):
    """
    Aggregate metrics across all test cases
    """

    pass_rate: float = pydantic.Field()
    """
    Percentage of tests passed (0-1)
    """

    accuracy: float = pydantic.Field()
    """
    Exact match accuracy (0-1)
    """

    avg_similarity_score: typing.Optional[float] = pydantic.Field(default=None)
    """
    Average semantic similarity score (0-1)
    """

    min_similarity_score: typing.Optional[float] = pydantic.Field(default=None)
    """
    Minimum similarity score across tests
    """

    max_similarity_score: typing.Optional[float] = pydantic.Field(default=None)
    """
    Maximum similarity score across tests
    """

    avg_correctness: typing.Optional[float] = pydantic.Field(default=None)
    """
    Average correctness score from LLM judge
    """

    avg_helpfulness: typing.Optional[float] = pydantic.Field(default=None)
    """
    Average helpfulness score from LLM judge
    """

    avg_coherence: typing.Optional[float] = pydantic.Field(default=None)
    """
    Average coherence score from LLM judge
    """

    judge_scores: typing.Optional[typing.Dict[str, typing.Optional[float]]] = pydantic.Field(default=None)
    """
    Scores for custom judge criteria
    """

    scorer_metrics: typing.Optional[typing.Dict[str, typing.Optional[typing.Dict[str, typing.Optional[float]]]]] = (
        pydantic.Field(default=None)
    )
    """
    Per-scorer aggregated metrics {scorer_id: {avg, min, max, pass_rate}}
    """

    avg_composite_score: typing.Optional[float] = pydantic.Field(default=None)
    """
    Average composite score across all tests (0-1)
    """

    dimension_metrics: typing.Optional[typing.Dict[str, typing.Optional[typing.Dict[str, typing.Optional[float]]]]] = (
        pydantic.Field(default=None)
    )
    """
    Per-dimension aggregated metrics {dimension_name: {avg, min, max, count}}
    """

    tag_metrics: typing.Optional[typing.Dict[str, typing.Optional[typing.Dict[str, typing.Optional[typing.Any]]]]] = (
        pydantic.Field(default=None)
    )
    """
    Metrics grouped by test case tags {tag: {count, pass_rate, ...}}
    """

    human_review_pending_count: typing.Optional[int] = pydantic.Field(default=None)
    """
    Number of test cases with pending human reviews
    """

    avg_latency_ms: float = pydantic.Field()
    """
    Average test latency
    """

    median_latency_ms: float = pydantic.Field()
    """
    Median test latency
    """

    p_95_latency_ms: typing_extensions.Annotated[float, FieldMetadata(alias="p95_latency_ms")] = pydantic.Field()
    """
    95th percentile latency
    """

    min_latency_ms: int = pydantic.Field()
    """
    Minimum test latency
    """

    max_latency_ms: int = pydantic.Field()
    """
    Maximum test latency
    """

    total_tokens: typing.Optional[int] = pydantic.Field(default=None)
    """
    Total tokens used across all tests
    """

    total_cost_usd: typing.Optional[float] = pydantic.Field(default=None)
    """
    Total cost in USD
    """

    pass_count: int = pydantic.Field()
    """
    Number of tests passed
    """

    fail_count: int = pydantic.Field()
    """
    Number of tests failed
    """

    error_count: int = pydantic.Field()
    """
    Number of tests with errors
    """

    total_count: int = pydantic.Field()
    """
    Total number of tests
    """

    if IS_PYDANTIC_V2:
        model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True)  # type: ignore # Pydantic v2
    else:

        class Config:
            frozen = True
            smart_union = True
            extra = pydantic.Extra.allow
