# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.

from __future__ import annotations

from typing import Dict, List, overload
from typing_extensions import Literal

import httpx

from ...types import (
    evaluation_dataset_list_params,
    evaluation_dataset_create_params,
    evaluation_dataset_update_params,
    evaluation_dataset_publish_params,
    evaluation_dataset_approve_batch_params,
)
from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven
from ..._utils import (
    required_args,
    maybe_transform,
    async_maybe_transform,
)
from ..._compat import cached_property
from .test_cases import (
    TestCasesResource,
    AsyncTestCasesResource,
    TestCasesResourceWithRawResponse,
    AsyncTestCasesResourceWithRawResponse,
    TestCasesResourceWithStreamingResponse,
    AsyncTestCasesResourceWithStreamingResponse,
)
from ..._resource import SyncAPIResource, AsyncAPIResource
from ..._response import (
    to_raw_response_wrapper,
    to_streamed_response_wrapper,
    async_to_raw_response_wrapper,
    async_to_streamed_response_wrapper,
)
from ...pagination import SyncPageResponse, AsyncPageResponse
from ..._base_client import AsyncPaginator, make_request_options
from .generation_jobs import (
    GenerationJobsResource,
    AsyncGenerationJobsResource,
    GenerationJobsResourceWithRawResponse,
    AsyncGenerationJobsResourceWithRawResponse,
    GenerationJobsResourceWithStreamingResponse,
    AsyncGenerationJobsResourceWithStreamingResponse,
)
from .test_cases.test_cases import TestCasesResource, AsyncTestCasesResource
from ...types.evaluation_dataset import EvaluationDataset
from .evaluation_dataset_versions import (
    EvaluationDatasetVersionsResource,
    AsyncEvaluationDatasetVersionsResource,
    EvaluationDatasetVersionsResourceWithRawResponse,
    AsyncEvaluationDatasetVersionsResourceWithRawResponse,
    EvaluationDatasetVersionsResourceWithStreamingResponse,
    AsyncEvaluationDatasetVersionsResourceWithStreamingResponse,
)
from .autogenerated_draft_test_cases import (
    AutogeneratedDraftTestCasesResource,
    AsyncAutogeneratedDraftTestCasesResource,
    AutogeneratedDraftTestCasesResourceWithRawResponse,
    AsyncAutogeneratedDraftTestCasesResourceWithRawResponse,
    AutogeneratedDraftTestCasesResourceWithStreamingResponse,
    AsyncAutogeneratedDraftTestCasesResourceWithStreamingResponse,
)
from ...types.shared.generic_delete_response import GenericDeleteResponse
from ...types.publish_evaluation_dataset_draft_response import PublishEvaluationDatasetDraftResponse
from ...types.auto_generated_draft_test_case_approve_batch_response import (
    AutoGeneratedDraftTestCaseApproveBatchResponse,
)

__all__ = ["EvaluationDatasetsResource", "AsyncEvaluationDatasetsResource"]


class EvaluationDatasetsResource(SyncAPIResource):
    @cached_property
    def evaluation_dataset_versions(self) -> EvaluationDatasetVersionsResource:
        return EvaluationDatasetVersionsResource(self._client)

    @cached_property
    def test_cases(self) -> TestCasesResource:
        return TestCasesResource(self._client)

    @cached_property
    def autogenerated_draft_test_cases(self) -> AutogeneratedDraftTestCasesResource:
        return AutogeneratedDraftTestCasesResource(self._client)

    @cached_property
    def generation_jobs(self) -> GenerationJobsResource:
        return GenerationJobsResource(self._client)

    @cached_property
    def with_raw_response(self) -> EvaluationDatasetsResourceWithRawResponse:
        """
        This property can be used as a prefix for any HTTP method call to return the
        the raw response object instead of the parsed content.

        For more information, see https://www.github.com/scaleapi/sgp-python#accessing-raw-response-data-eg-headers
        """
        return EvaluationDatasetsResourceWithRawResponse(self)

    @cached_property
    def with_streaming_response(self) -> EvaluationDatasetsResourceWithStreamingResponse:
        """
        An alternative to `.with_raw_response` that doesn't eagerly read the response body.

        For more information, see https://www.github.com/scaleapi/sgp-python#with_streaming_response
        """
        return EvaluationDatasetsResourceWithStreamingResponse(self)

    @overload
    def create(
        self,
        *,
        account_id: str,
        name: str,
        schema_type: Literal["GENERATION", "FLEXIBLE"],
        type: Literal["manual"] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Creates a evaluation dataset

        ### Details

        This API can be used to create a evaluation dataset. To use this API, review the
        request schema and pass in all fields that are required to create a evaluation
        dataset.

        Args:
          account_id: The ID of the account that owns the given entity.

          name: The name of the dataset

          schema_type: An enumeration.

          type: Manually or automatically generated dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @overload
    def create(
        self,
        *,
        account_id: str,
        knowledge_base_id: str,
        name: str,
        schema_type: Literal["GENERATION"],
        type: Literal["autogenerated"] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Creates a evaluation dataset

        ### Details

        This API can be used to create a evaluation dataset. To use this API, review the
        request schema and pass in all fields that are required to create a evaluation
        dataset.

        Args:
          account_id: The ID of the account that owns the given entity.

          knowledge_base_id: ID of the knowledge base to generate the evaluation dataset from.

          name: The name of the dataset

          schema_type: The schema type of the dataset. Only GENERATION is currently supported.

          type: Manually or automatically generated dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @overload
    def create(
        self,
        *,
        account_id: str,
        advanced_config: Dict[str, List[str]],
        harms_list: List[str],
        name: str,
        schema_type: Literal["GENERATION"],
        type: Literal["safety"] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Creates a evaluation dataset

        ### Details

        This API can be used to create a evaluation dataset. To use this API, review the
        request schema and pass in all fields that are required to create a evaluation
        dataset.

        Args:
          account_id: The ID of the account that owns the given entity.

          advanced_config: Advanced configuration for the safety evaluation dataset.

          harms_list: List of harms to generate the dataset for.

          name: The name of the dataset

          schema_type: The schema type of the dataset. Only GENERATION is currently supported.

          type: Manually or automatically generated dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @required_args(
        ["account_id", "name", "schema_type"],
        ["account_id", "knowledge_base_id", "name", "schema_type"],
        ["account_id", "advanced_config", "harms_list", "name", "schema_type"],
    )
    def create(
        self,
        *,
        account_id: str,
        name: str,
        schema_type: Literal["GENERATION", "FLEXIBLE"] | Literal["GENERATION"],
        type: Literal["manual"] | Literal["autogenerated"] | Literal["safety"] | NotGiven = NOT_GIVEN,
        knowledge_base_id: str | NotGiven = NOT_GIVEN,
        advanced_config: Dict[str, List[str]] | NotGiven = NOT_GIVEN,
        harms_list: List[str] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        return self._post(
            "/v4/evaluation-datasets",
            body=maybe_transform(
                {
                    "account_id": account_id,
                    "name": name,
                    "schema_type": schema_type,
                    "type": type,
                    "knowledge_base_id": knowledge_base_id,
                    "advanced_config": advanced_config,
                    "harms_list": harms_list,
                },
                evaluation_dataset_create_params.EvaluationDatasetCreateParams,
            ),
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=EvaluationDataset,
        )

    def retrieve(
        self,
        evaluation_dataset_id: str,
        *,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Gets the details of a dataset.

        Args:
          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return self._get(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}",
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=EvaluationDataset,
        )

    @overload
    def update(
        self,
        evaluation_dataset_id: str,
        *,
        name: str,
        restore: Literal[False] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Updates a evaluation dataset

        ### Details

        This API can be used to update the evaluation dataset that matches the ID that
        was passed in as a path parameter. To use this API, pass in the `id` that was
        returned from your Create Evaluation Dataset API call as a path parameter.

        Review the request schema to see the fields that can be updated.

        Args:
          name: The name of the dataset

          restore: Set to true to restore the entity from the database.

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @overload
    def update(
        self,
        evaluation_dataset_id: str,
        *,
        restore: Literal[True],
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Updates a evaluation dataset

        ### Details

        This API can be used to update the evaluation dataset that matches the ID that
        was passed in as a path parameter. To use this API, pass in the `id` that was
        returned from your Create Evaluation Dataset API call as a path parameter.

        Review the request schema to see the fields that can be updated.

        Args:
          restore: Set to true to restore the entity from the database.

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @required_args(["name"], ["restore"])
    def update(
        self,
        evaluation_dataset_id: str,
        *,
        name: str | NotGiven = NOT_GIVEN,
        restore: Literal[False] | Literal[True] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return self._patch(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}",
            body=maybe_transform(
                {
                    "name": name,
                    "restore": restore,
                },
                evaluation_dataset_update_params.EvaluationDatasetUpdateParams,
            ),
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=EvaluationDataset,
        )

    def list(
        self,
        *,
        account_id: str | NotGiven = NOT_GIVEN,
        include_archived: bool | NotGiven = NOT_GIVEN,
        limit: int | NotGiven = NOT_GIVEN,
        page: int | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> SyncPageResponse[EvaluationDataset]:
        """
        ### Description

        Lists all evaluation datasets accessible to the user.

        ### Details

        This API can be used to list evaluation datasets. If a user has access to
        multiple accounts, all evaluation datasets from all accounts the user is
        associated with will be returned.

        Args:
          limit: Maximum number of artifacts to be returned by the given endpoint. Defaults to
              100 and cannot be greater than 10k.

          page: Page number for pagination to be returned by the given endpoint. Starts at page
              1

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        return self._get_api_list(
            "/v4/evaluation-datasets",
            page=SyncPageResponse[EvaluationDataset],
            options=make_request_options(
                extra_headers=extra_headers,
                extra_query=extra_query,
                extra_body=extra_body,
                timeout=timeout,
                query=maybe_transform(
                    {
                        "account_id": account_id,
                        "include_archived": include_archived,
                        "limit": limit,
                        "page": page,
                    },
                    evaluation_dataset_list_params.EvaluationDatasetListParams,
                ),
            ),
            model=EvaluationDataset,
        )

    def approve_batch(
        self,
        evaluation_dataset_id: str,
        *,
        autogenerated_draft_test_cases: List[str],
        force: bool | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> AutoGeneratedDraftTestCaseApproveBatchResponse:
        """
        Approve Auto Generated Test Cases Batch

        Args:
          autogenerated_draft_test_cases: Ids of auto generated draft test cases to be approved.

          force: Force approve a batch of autogenerated test case IDs for the evaluation dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return self._post(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}/approve-batch",
            body=maybe_transform(
                {"autogenerated_draft_test_cases": autogenerated_draft_test_cases},
                evaluation_dataset_approve_batch_params.EvaluationDatasetApproveBatchParams,
            ),
            options=make_request_options(
                extra_headers=extra_headers,
                extra_query=extra_query,
                extra_body=extra_body,
                timeout=timeout,
                query=maybe_transform(
                    {"force": force}, evaluation_dataset_approve_batch_params.EvaluationDatasetApproveBatchParams
                ),
            ),
            cast_to=AutoGeneratedDraftTestCaseApproveBatchResponse,
        )

    def publish(
        self,
        evaluation_dataset_id: str,
        *,
        force: bool | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> PublishEvaluationDatasetDraftResponse:
        """
        Publish Evaluation Dataset

        Args:
          force: Force approve an evaluation dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return self._post(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}/publish",
            options=make_request_options(
                extra_headers=extra_headers,
                extra_query=extra_query,
                extra_body=extra_body,
                timeout=timeout,
                query=maybe_transform(
                    {"force": force}, evaluation_dataset_publish_params.EvaluationDatasetPublishParams
                ),
            ),
            cast_to=PublishEvaluationDatasetDraftResponse,
        )

    def remove(
        self,
        evaluation_dataset_id: str,
        *,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> GenericDeleteResponse:
        """
        ### Description

        Deletes the dataset, and all other entities associated with the dataset, such as
        test cases, evaluations and results.

        ### Details

        **This is a permanent and destructive action that cannot be undone.**

        Args:
          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return self._delete(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}",
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=GenericDeleteResponse,
        )


class AsyncEvaluationDatasetsResource(AsyncAPIResource):
    @cached_property
    def evaluation_dataset_versions(self) -> AsyncEvaluationDatasetVersionsResource:
        return AsyncEvaluationDatasetVersionsResource(self._client)

    @cached_property
    def test_cases(self) -> AsyncTestCasesResource:
        return AsyncTestCasesResource(self._client)

    @cached_property
    def autogenerated_draft_test_cases(self) -> AsyncAutogeneratedDraftTestCasesResource:
        return AsyncAutogeneratedDraftTestCasesResource(self._client)

    @cached_property
    def generation_jobs(self) -> AsyncGenerationJobsResource:
        return AsyncGenerationJobsResource(self._client)

    @cached_property
    def with_raw_response(self) -> AsyncEvaluationDatasetsResourceWithRawResponse:
        """
        This property can be used as a prefix for any HTTP method call to return the
        the raw response object instead of the parsed content.

        For more information, see https://www.github.com/scaleapi/sgp-python#accessing-raw-response-data-eg-headers
        """
        return AsyncEvaluationDatasetsResourceWithRawResponse(self)

    @cached_property
    def with_streaming_response(self) -> AsyncEvaluationDatasetsResourceWithStreamingResponse:
        """
        An alternative to `.with_raw_response` that doesn't eagerly read the response body.

        For more information, see https://www.github.com/scaleapi/sgp-python#with_streaming_response
        """
        return AsyncEvaluationDatasetsResourceWithStreamingResponse(self)

    @overload
    async def create(
        self,
        *,
        account_id: str,
        name: str,
        schema_type: Literal["GENERATION", "FLEXIBLE"],
        type: Literal["manual"] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Creates a evaluation dataset

        ### Details

        This API can be used to create a evaluation dataset. To use this API, review the
        request schema and pass in all fields that are required to create a evaluation
        dataset.

        Args:
          account_id: The ID of the account that owns the given entity.

          name: The name of the dataset

          schema_type: An enumeration.

          type: Manually or automatically generated dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @overload
    async def create(
        self,
        *,
        account_id: str,
        knowledge_base_id: str,
        name: str,
        schema_type: Literal["GENERATION"],
        type: Literal["autogenerated"] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Creates a evaluation dataset

        ### Details

        This API can be used to create a evaluation dataset. To use this API, review the
        request schema and pass in all fields that are required to create a evaluation
        dataset.

        Args:
          account_id: The ID of the account that owns the given entity.

          knowledge_base_id: ID of the knowledge base to generate the evaluation dataset from.

          name: The name of the dataset

          schema_type: The schema type of the dataset. Only GENERATION is currently supported.

          type: Manually or automatically generated dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @overload
    async def create(
        self,
        *,
        account_id: str,
        advanced_config: Dict[str, List[str]],
        harms_list: List[str],
        name: str,
        schema_type: Literal["GENERATION"],
        type: Literal["safety"] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Creates a evaluation dataset

        ### Details

        This API can be used to create a evaluation dataset. To use this API, review the
        request schema and pass in all fields that are required to create a evaluation
        dataset.

        Args:
          account_id: The ID of the account that owns the given entity.

          advanced_config: Advanced configuration for the safety evaluation dataset.

          harms_list: List of harms to generate the dataset for.

          name: The name of the dataset

          schema_type: The schema type of the dataset. Only GENERATION is currently supported.

          type: Manually or automatically generated dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @required_args(
        ["account_id", "name", "schema_type"],
        ["account_id", "knowledge_base_id", "name", "schema_type"],
        ["account_id", "advanced_config", "harms_list", "name", "schema_type"],
    )
    async def create(
        self,
        *,
        account_id: str,
        name: str,
        schema_type: Literal["GENERATION", "FLEXIBLE"] | Literal["GENERATION"],
        type: Literal["manual"] | Literal["autogenerated"] | Literal["safety"] | NotGiven = NOT_GIVEN,
        knowledge_base_id: str | NotGiven = NOT_GIVEN,
        advanced_config: Dict[str, List[str]] | NotGiven = NOT_GIVEN,
        harms_list: List[str] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        return await self._post(
            "/v4/evaluation-datasets",
            body=await async_maybe_transform(
                {
                    "account_id": account_id,
                    "name": name,
                    "schema_type": schema_type,
                    "type": type,
                    "knowledge_base_id": knowledge_base_id,
                    "advanced_config": advanced_config,
                    "harms_list": harms_list,
                },
                evaluation_dataset_create_params.EvaluationDatasetCreateParams,
            ),
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=EvaluationDataset,
        )

    async def retrieve(
        self,
        evaluation_dataset_id: str,
        *,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Gets the details of a dataset.

        Args:
          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return await self._get(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}",
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=EvaluationDataset,
        )

    @overload
    async def update(
        self,
        evaluation_dataset_id: str,
        *,
        name: str,
        restore: Literal[False] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Updates a evaluation dataset

        ### Details

        This API can be used to update the evaluation dataset that matches the ID that
        was passed in as a path parameter. To use this API, pass in the `id` that was
        returned from your Create Evaluation Dataset API call as a path parameter.

        Review the request schema to see the fields that can be updated.

        Args:
          name: The name of the dataset

          restore: Set to true to restore the entity from the database.

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @overload
    async def update(
        self,
        evaluation_dataset_id: str,
        *,
        restore: Literal[True],
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        """
        ### Description

        Updates a evaluation dataset

        ### Details

        This API can be used to update the evaluation dataset that matches the ID that
        was passed in as a path parameter. To use this API, pass in the `id` that was
        returned from your Create Evaluation Dataset API call as a path parameter.

        Review the request schema to see the fields that can be updated.

        Args:
          restore: Set to true to restore the entity from the database.

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        ...

    @required_args(["name"], ["restore"])
    async def update(
        self,
        evaluation_dataset_id: str,
        *,
        name: str | NotGiven = NOT_GIVEN,
        restore: Literal[False] | Literal[True] | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> EvaluationDataset:
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return await self._patch(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}",
            body=await async_maybe_transform(
                {
                    "name": name,
                    "restore": restore,
                },
                evaluation_dataset_update_params.EvaluationDatasetUpdateParams,
            ),
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=EvaluationDataset,
        )

    def list(
        self,
        *,
        account_id: str | NotGiven = NOT_GIVEN,
        include_archived: bool | NotGiven = NOT_GIVEN,
        limit: int | NotGiven = NOT_GIVEN,
        page: int | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> AsyncPaginator[EvaluationDataset, AsyncPageResponse[EvaluationDataset]]:
        """
        ### Description

        Lists all evaluation datasets accessible to the user.

        ### Details

        This API can be used to list evaluation datasets. If a user has access to
        multiple accounts, all evaluation datasets from all accounts the user is
        associated with will be returned.

        Args:
          limit: Maximum number of artifacts to be returned by the given endpoint. Defaults to
              100 and cannot be greater than 10k.

          page: Page number for pagination to be returned by the given endpoint. Starts at page
              1

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        return self._get_api_list(
            "/v4/evaluation-datasets",
            page=AsyncPageResponse[EvaluationDataset],
            options=make_request_options(
                extra_headers=extra_headers,
                extra_query=extra_query,
                extra_body=extra_body,
                timeout=timeout,
                query=maybe_transform(
                    {
                        "account_id": account_id,
                        "include_archived": include_archived,
                        "limit": limit,
                        "page": page,
                    },
                    evaluation_dataset_list_params.EvaluationDatasetListParams,
                ),
            ),
            model=EvaluationDataset,
        )

    async def approve_batch(
        self,
        evaluation_dataset_id: str,
        *,
        autogenerated_draft_test_cases: List[str],
        force: bool | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> AutoGeneratedDraftTestCaseApproveBatchResponse:
        """
        Approve Auto Generated Test Cases Batch

        Args:
          autogenerated_draft_test_cases: Ids of auto generated draft test cases to be approved.

          force: Force approve a batch of autogenerated test case IDs for the evaluation dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return await self._post(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}/approve-batch",
            body=await async_maybe_transform(
                {"autogenerated_draft_test_cases": autogenerated_draft_test_cases},
                evaluation_dataset_approve_batch_params.EvaluationDatasetApproveBatchParams,
            ),
            options=make_request_options(
                extra_headers=extra_headers,
                extra_query=extra_query,
                extra_body=extra_body,
                timeout=timeout,
                query=await async_maybe_transform(
                    {"force": force}, evaluation_dataset_approve_batch_params.EvaluationDatasetApproveBatchParams
                ),
            ),
            cast_to=AutoGeneratedDraftTestCaseApproveBatchResponse,
        )

    async def publish(
        self,
        evaluation_dataset_id: str,
        *,
        force: bool | NotGiven = NOT_GIVEN,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> PublishEvaluationDatasetDraftResponse:
        """
        Publish Evaluation Dataset

        Args:
          force: Force approve an evaluation dataset

          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return await self._post(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}/publish",
            options=make_request_options(
                extra_headers=extra_headers,
                extra_query=extra_query,
                extra_body=extra_body,
                timeout=timeout,
                query=await async_maybe_transform(
                    {"force": force}, evaluation_dataset_publish_params.EvaluationDatasetPublishParams
                ),
            ),
            cast_to=PublishEvaluationDatasetDraftResponse,
        )

    async def remove(
        self,
        evaluation_dataset_id: str,
        *,
        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
        # The extra values given here take precedence over values defined on the client or passed to this method.
        extra_headers: Headers | None = None,
        extra_query: Query | None = None,
        extra_body: Body | None = None,
        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
    ) -> GenericDeleteResponse:
        """
        ### Description

        Deletes the dataset, and all other entities associated with the dataset, such as
        test cases, evaluations and results.

        ### Details

        **This is a permanent and destructive action that cannot be undone.**

        Args:
          extra_headers: Send extra headers

          extra_query: Add additional query parameters to the request

          extra_body: Add additional JSON properties to the request

          timeout: Override the client-level default timeout for this request, in seconds
        """
        if not evaluation_dataset_id:
            raise ValueError(
                f"Expected a non-empty value for `evaluation_dataset_id` but received {evaluation_dataset_id!r}"
            )
        return await self._delete(
            f"/v4/evaluation-datasets/{evaluation_dataset_id}",
            options=make_request_options(
                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
            ),
            cast_to=GenericDeleteResponse,
        )


class EvaluationDatasetsResourceWithRawResponse:
    def __init__(self, evaluation_datasets: EvaluationDatasetsResource) -> None:
        self._evaluation_datasets = evaluation_datasets

        self.create = to_raw_response_wrapper(
            evaluation_datasets.create,
        )
        self.retrieve = to_raw_response_wrapper(
            evaluation_datasets.retrieve,
        )
        self.update = to_raw_response_wrapper(
            evaluation_datasets.update,
        )
        self.list = to_raw_response_wrapper(
            evaluation_datasets.list,
        )
        self.approve_batch = to_raw_response_wrapper(
            evaluation_datasets.approve_batch,
        )
        self.publish = to_raw_response_wrapper(
            evaluation_datasets.publish,
        )
        self.remove = to_raw_response_wrapper(
            evaluation_datasets.remove,
        )

    @cached_property
    def evaluation_dataset_versions(self) -> EvaluationDatasetVersionsResourceWithRawResponse:
        return EvaluationDatasetVersionsResourceWithRawResponse(self._evaluation_datasets.evaluation_dataset_versions)

    @cached_property
    def test_cases(self) -> TestCasesResourceWithRawResponse:
        return TestCasesResourceWithRawResponse(self._evaluation_datasets.test_cases)

    @cached_property
    def autogenerated_draft_test_cases(self) -> AutogeneratedDraftTestCasesResourceWithRawResponse:
        return AutogeneratedDraftTestCasesResourceWithRawResponse(
            self._evaluation_datasets.autogenerated_draft_test_cases
        )

    @cached_property
    def generation_jobs(self) -> GenerationJobsResourceWithRawResponse:
        return GenerationJobsResourceWithRawResponse(self._evaluation_datasets.generation_jobs)


class AsyncEvaluationDatasetsResourceWithRawResponse:
    def __init__(self, evaluation_datasets: AsyncEvaluationDatasetsResource) -> None:
        self._evaluation_datasets = evaluation_datasets

        self.create = async_to_raw_response_wrapper(
            evaluation_datasets.create,
        )
        self.retrieve = async_to_raw_response_wrapper(
            evaluation_datasets.retrieve,
        )
        self.update = async_to_raw_response_wrapper(
            evaluation_datasets.update,
        )
        self.list = async_to_raw_response_wrapper(
            evaluation_datasets.list,
        )
        self.approve_batch = async_to_raw_response_wrapper(
            evaluation_datasets.approve_batch,
        )
        self.publish = async_to_raw_response_wrapper(
            evaluation_datasets.publish,
        )
        self.remove = async_to_raw_response_wrapper(
            evaluation_datasets.remove,
        )

    @cached_property
    def evaluation_dataset_versions(self) -> AsyncEvaluationDatasetVersionsResourceWithRawResponse:
        return AsyncEvaluationDatasetVersionsResourceWithRawResponse(
            self._evaluation_datasets.evaluation_dataset_versions
        )

    @cached_property
    def test_cases(self) -> AsyncTestCasesResourceWithRawResponse:
        return AsyncTestCasesResourceWithRawResponse(self._evaluation_datasets.test_cases)

    @cached_property
    def autogenerated_draft_test_cases(self) -> AsyncAutogeneratedDraftTestCasesResourceWithRawResponse:
        return AsyncAutogeneratedDraftTestCasesResourceWithRawResponse(
            self._evaluation_datasets.autogenerated_draft_test_cases
        )

    @cached_property
    def generation_jobs(self) -> AsyncGenerationJobsResourceWithRawResponse:
        return AsyncGenerationJobsResourceWithRawResponse(self._evaluation_datasets.generation_jobs)


class EvaluationDatasetsResourceWithStreamingResponse:
    def __init__(self, evaluation_datasets: EvaluationDatasetsResource) -> None:
        self._evaluation_datasets = evaluation_datasets

        self.create = to_streamed_response_wrapper(
            evaluation_datasets.create,
        )
        self.retrieve = to_streamed_response_wrapper(
            evaluation_datasets.retrieve,
        )
        self.update = to_streamed_response_wrapper(
            evaluation_datasets.update,
        )
        self.list = to_streamed_response_wrapper(
            evaluation_datasets.list,
        )
        self.approve_batch = to_streamed_response_wrapper(
            evaluation_datasets.approve_batch,
        )
        self.publish = to_streamed_response_wrapper(
            evaluation_datasets.publish,
        )
        self.remove = to_streamed_response_wrapper(
            evaluation_datasets.remove,
        )

    @cached_property
    def evaluation_dataset_versions(self) -> EvaluationDatasetVersionsResourceWithStreamingResponse:
        return EvaluationDatasetVersionsResourceWithStreamingResponse(
            self._evaluation_datasets.evaluation_dataset_versions
        )

    @cached_property
    def test_cases(self) -> TestCasesResourceWithStreamingResponse:
        return TestCasesResourceWithStreamingResponse(self._evaluation_datasets.test_cases)

    @cached_property
    def autogenerated_draft_test_cases(self) -> AutogeneratedDraftTestCasesResourceWithStreamingResponse:
        return AutogeneratedDraftTestCasesResourceWithStreamingResponse(
            self._evaluation_datasets.autogenerated_draft_test_cases
        )

    @cached_property
    def generation_jobs(self) -> GenerationJobsResourceWithStreamingResponse:
        return GenerationJobsResourceWithStreamingResponse(self._evaluation_datasets.generation_jobs)


class AsyncEvaluationDatasetsResourceWithStreamingResponse:
    def __init__(self, evaluation_datasets: AsyncEvaluationDatasetsResource) -> None:
        self._evaluation_datasets = evaluation_datasets

        self.create = async_to_streamed_response_wrapper(
            evaluation_datasets.create,
        )
        self.retrieve = async_to_streamed_response_wrapper(
            evaluation_datasets.retrieve,
        )
        self.update = async_to_streamed_response_wrapper(
            evaluation_datasets.update,
        )
        self.list = async_to_streamed_response_wrapper(
            evaluation_datasets.list,
        )
        self.approve_batch = async_to_streamed_response_wrapper(
            evaluation_datasets.approve_batch,
        )
        self.publish = async_to_streamed_response_wrapper(
            evaluation_datasets.publish,
        )
        self.remove = async_to_streamed_response_wrapper(
            evaluation_datasets.remove,
        )

    @cached_property
    def evaluation_dataset_versions(self) -> AsyncEvaluationDatasetVersionsResourceWithStreamingResponse:
        return AsyncEvaluationDatasetVersionsResourceWithStreamingResponse(
            self._evaluation_datasets.evaluation_dataset_versions
        )

    @cached_property
    def test_cases(self) -> AsyncTestCasesResourceWithStreamingResponse:
        return AsyncTestCasesResourceWithStreamingResponse(self._evaluation_datasets.test_cases)

    @cached_property
    def autogenerated_draft_test_cases(self) -> AsyncAutogeneratedDraftTestCasesResourceWithStreamingResponse:
        return AsyncAutogeneratedDraftTestCasesResourceWithStreamingResponse(
            self._evaluation_datasets.autogenerated_draft_test_cases
        )

    @cached_property
    def generation_jobs(self) -> AsyncGenerationJobsResourceWithStreamingResponse:
        return AsyncGenerationJobsResourceWithStreamingResponse(self._evaluation_datasets.generation_jobs)
