# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.

from typing import List, Union, Optional
from datetime import datetime
from typing_extensions import Literal, TypeAlias

from pydantic import Field as FieldInfo

from .._models import BaseModel
from .eval_definition_output import EvalDefinitionOutput

__all__ = [
    "EvalOutput",
    "Results",
    "ResultsUnionMember0",
    "ResultsUnionMember1",
    "ResultsUnionMember2",
    "ResultsUnionMember2AnswerRelevancy",
    "ResultsUnionMember2AnswerRelevancyMetadata",
    "ResultsUnionMember2AnswerRelevancyMetadataQuestion",
    "ResultsUnionMember2AnswerRelevancyMetadataSimilarity",
    "ResultsUnionMember2ContextPrecision",
    "ResultsUnionMember2ContextPrecisionMetadata",
    "ResultsUnionMember2ContextRelevancy",
    "ResultsUnionMember2ContextRelevancyMetadata",
    "ResultsUnionMember2ContextRelevancyMetadataRelevantSentence",
    "ResultsUnionMember2Faithfulness",
    "ResultsUnionMember2FaithfulnessMetadata",
    "ResultsUnionMember2FaithfulnessMetadataFaithfulness",
    "ResultsUnionMember3",
    "ResultsUnionMember3Metadata",
    "ResultsUnionMember4",
    "ResultsUnionMember4Classification",
    "ResultsUnionMember5",
    "ResultsUnionMember5Missing",
]


class ResultsUnionMember0(BaseModel):
    analysis: str

    clarity: float

    coherence: float

    engagingness: float

    naturalness: float

    relevance: float


class ResultsUnionMember1(BaseModel):
    analysis: str
    """
    A brief explanation for your rating, referring to specific aspects of the
    response and the query. Make sure that the explanation is formatted as markdown,
    and that it is easy to read and understand.
    """

    score: float
    """The score of the response based on the style guide from 1-5"""


class ResultsUnionMember2AnswerRelevancyMetadataQuestion(BaseModel):
    question: str


class ResultsUnionMember2AnswerRelevancyMetadataSimilarity(BaseModel):
    question: str

    score: float


class ResultsUnionMember2AnswerRelevancyMetadata(BaseModel):
    questions: List[ResultsUnionMember2AnswerRelevancyMetadataQuestion]

    similarity: List[ResultsUnionMember2AnswerRelevancyMetadataSimilarity]


class ResultsUnionMember2AnswerRelevancy(BaseModel):
    name: str

    score: float

    error: Optional[str] = None

    metadata: Optional[ResultsUnionMember2AnswerRelevancyMetadata] = None


class ResultsUnionMember2ContextPrecisionMetadata(BaseModel):
    reason: str

    verdict: int


class ResultsUnionMember2ContextPrecision(BaseModel):
    name: str

    score: float

    error: Optional[str] = None

    metadata: Optional[ResultsUnionMember2ContextPrecisionMetadata] = None


class ResultsUnionMember2ContextRelevancyMetadataRelevantSentence(BaseModel):
    reasons: List[str]

    sentence: str


class ResultsUnionMember2ContextRelevancyMetadata(BaseModel):
    relevant_sentences: List[ResultsUnionMember2ContextRelevancyMetadataRelevantSentence] = FieldInfo(
        alias="relevantSentences"
    )


class ResultsUnionMember2ContextRelevancy(BaseModel):
    name: str

    score: float

    error: Optional[str] = None

    metadata: Optional[ResultsUnionMember2ContextRelevancyMetadata] = None


class ResultsUnionMember2FaithfulnessMetadataFaithfulness(BaseModel):
    reason: str

    statement: str

    verdict: int

    classification: Optional[Literal["UNSUPPORTED_CLAIM", "CONTRADICTION", "PARTIAL_HALLUCINATION", "SCOPE_DRIFT"]] = (
        None
    )
    """Classification of the hallucination type (only present when verdict is 0)"""


class ResultsUnionMember2FaithfulnessMetadata(BaseModel):
    faithfulness: List[ResultsUnionMember2FaithfulnessMetadataFaithfulness]

    statements: List[str]


class ResultsUnionMember2Faithfulness(BaseModel):
    name: str

    score: float

    error: Optional[str] = None

    metadata: Optional[ResultsUnionMember2FaithfulnessMetadata] = None


class ResultsUnionMember2(BaseModel):
    answer_relevancy: ResultsUnionMember2AnswerRelevancy = FieldInfo(alias="AnswerRelevancy")

    context_precision: ResultsUnionMember2ContextPrecision = FieldInfo(alias="ContextPrecision")

    context_relevancy: ResultsUnionMember2ContextRelevancy = FieldInfo(alias="ContextRelevancy")

    faithfulness: ResultsUnionMember2Faithfulness = FieldInfo(alias="Faithfulness")


class ResultsUnionMember3Metadata(BaseModel):
    rationale: str


class ResultsUnionMember3(BaseModel):
    name: str

    score: int

    error: Optional[str] = None

    metadata: Optional[ResultsUnionMember3Metadata] = None


class ResultsUnionMember4Classification(BaseModel):
    fn: List[str] = FieldInfo(alias="FN")
    """False negatives: Statements found in the ground truth but omitted in the answer"""

    fp: List[str] = FieldInfo(alias="FP")
    """
    False positives: Statements present in the answer but not found in the ground
    truth
    """

    tp: List[str] = FieldInfo(alias="TP")
    """
    True positives: Statements that are present in both the answer and the ground
    truth
    """


class ResultsUnionMember4(BaseModel):
    classification: ResultsUnionMember4Classification

    score: float
    """The F1 score of the response based on the fact checker (0-1)"""

    error: Optional[str] = None

    metadata: Optional[object] = None


class ResultsUnionMember5Missing(BaseModel):
    have: float

    need: float

    value: str


class ResultsUnionMember5(BaseModel):
    reason: str

    score: float

    expected: Optional[str] = None

    expected_list: Optional[List[str]] = FieldInfo(alias="expectedList", default=None)

    f1: Optional[float] = None

    got: Optional[str] = None

    got_list: Optional[List[str]] = FieldInfo(alias="gotList", default=None)

    match_mode: Optional[Literal["exact_unordered", "contains"]] = FieldInfo(alias="matchMode", default=None)

    missing: Optional[List[ResultsUnionMember5Missing]] = None

    precision: Optional[float] = None

    recall: Optional[float] = None

    score_metric: Optional[Literal["f1", "precision", "recall"]] = FieldInfo(alias="scoreMetric", default=None)

    tp: Optional[float] = None


Results: TypeAlias = Union[
    ResultsUnionMember0,
    ResultsUnionMember1,
    ResultsUnionMember2,
    ResultsUnionMember3,
    ResultsUnionMember4,
    ResultsUnionMember5,
]


class EvalOutput(BaseModel):
    """Complete evaluation information"""

    id: str
    """Unique identifier of the evaluation"""

    created_at: datetime = FieldInfo(alias="createdAt")
    """When the evaluation was created"""

    definition: EvalDefinitionOutput

    modified_at: datetime = FieldInfo(alias="modifiedAt")
    """When the evaluation was last modified"""

    org_id: str = FieldInfo(alias="orgId")
    """Organization ID that owns this evaluation"""

    status: Literal["PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"]
    """Status of the evaluation/test"""

    passed: Optional[bool] = None
    """Whether the evaluation passed"""

    results: Optional[Results] = None
    """Results of the evaluation (structure depends on eval type)."""

    score: Optional[float] = None
    """Overall score of the evaluation"""
