# This file defines all the model deployments that are supported by the Helm API.
# Some models have several deployments, each with different parameters.

# If you want to add a new deployment, you can technically do it here but we recommend
# you to do it in prod_env/model_deployments.yaml instead.

# Follow the template of this file to add a new deployment. You can copy paste this to get started:
#    # This file defines all the model deployments that you do not want to be public.
#    model_deployments: [] # Leave empty to disable private model deployments

model_deployments:
  - name: simple/model1
    model_name: simple/model1
    tokenizer_name: simple/tokenizer1
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.simple_client.SimpleClient"

  # Stanford Health Care
  # For internal use only for MedHELM
  # Placed earlier in the file to make them non-default
  - name: stanfordhealthcare/claude-3-5-sonnet-20241022
    model_name: anthropic/claude-3-5-sonnet-20241022
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: anthropic.claude-3-5-sonnet-20241022-v2:0
        deployment: Claude35Sonnetv2/awssig4fa
  
  - name: stanfordhealthcare/claude-3-7-sonnet-20250219
    model_name: anthropic/claude-3-7-sonnet-20250219
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0
        deployment: awssig4claude37/aswsig4claude37

  - name: stanfordhealthcare/claude-sonnet-4-20250514
    model_name: anthropic/claude-sonnet-4-20250514
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: us.anthropic.claude-sonnet-4-20250514-v1:0
        deployment: aws-claude4-sonnet/aws-claude4-sonnet
    
  - name: stanfordhealthcare/claude-sonnet-4-5-20250929
    model_name: anthropic/claude-sonnet-4-5-20250929
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0
        deployment: awssig4claude45sonnet/awssig4claude45sonnet

  - name: stanfordhealthcare/claude-haiku-4-5-20251001
    model_name: anthropic/claude-haiku-4-5-20251001
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-haiku-4-5-20251001-v1:0
        deployment: awssig4claude45haiku/awssig4claude45haiku

  - name: stanfordhealthcare/claude-opus-4-5-20251124
    model_name: anthropic/claude-opus-4-5-20251124
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-opus-4-5-20251124-v1:0
        deployment: awssig4claude45opus/awssig4claude45opus

  - name: stanfordhealthcare/nova-2-pro-v1:0
    model_name: amazon/nova-2-pro-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.amazon.nova-2-pro-v1:0
        deployment: awssig4nova2pro/awssig4nova2pro

  - name: stanfordhealthcare/nova-2-lite-v1:0
    model_name: amazon/nova-2-lite-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.amazon.nova-2-lite-v1:0
        deployment: awssig4nova2lite/awssig4nova2lite

  - name: stanfordhealthcare/gemini-1.5-pro-001
    model_name: google/gemini-1.5-pro-001
    tokenizer_name: google/gemma-2b
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
      args:
        deployment: gcpgemini/apim-gcp-oauth-fa

  - name: stanfordhealthcare/gemini-2.0-flash-001
    model_name: google/gemini-2.0-flash-001
    tokenizer_name: google/gemma-2b
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
      args:
        deployment: gcp-gem20flash-fa/apim-gcp-gem20flash-fa

  - name: stanfordhealthcare/gemini-2.5-pro-preview-05-06
    model_name: google/gemini-2.5-pro-preview-05-06
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576   # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
      args:
        deployment: gemini-25-pro/gemini-25-pro

  - name: stanfordhealthcare/gpt-4o-mini-2024-07-18
    model_name: openai/gpt-4o-mini-2024-07-18
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4o-mini
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4o-2024-05-13
    model_name: openai/gpt-4o-2024-05-13
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4o
        api_version: 2023-05-15
  
  - name: stanfordhealthcare/gpt-4-0613
    model_name: openai/gpt-4-0613
    tokenizer_name: openai/o200k_base
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4-turbo-2024-04-09
    model_name: openai/gpt-4-turbo-2024-04-09
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4-turbo
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4.1-2025-04-14
    model_name: openai/gpt-4.1-2025-04-14
    tokenizer_name: openai/o200k_base
    max_sequence_length: 1047576
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4.1
        api_version: 2025-01-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/o3-mini-2025-01-31
    model_name: openai/o3-mini-2025-01-31
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: o3-mini
        api_version: 2024-12-01-preview
        base_url: "{endpoint}/openai-eastus2"
  
  - name: stanfordhealthcare/gpt-5-2025-08-07
    model_name: openai/gpt-5-2025-08-07
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-5
        api_version: 2024-12-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/gpt-5-mini-2025-08-07
    model_name: openai/gpt-5-mini-2025-08-07
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-5-mini
        api_version: 2024-12-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/o4-mini-2025-04-16
    model_name: openai/o4-mini-2025-04-16
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o4-mini
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: o4-mini
        api_version: 2024-12-01-preview

  - name: stanfordhealthcare/o1-2024-12-17
    model_name: openai/o1-2024-12-17
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: o1
        api_version: 2024-12-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/deepseek-r1
    model_name: deepseek-ai/deepseek-r1
    tokenizer_name: deepseek-ai/deepseek-r1
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        openai_model_name: deepseek-chat
        output_processor: helm.benchmark.metrics.output_processors.remove_deepseek_r1_thinking
        base_url: "{endpoint}/deepseekr1/v1"

  - name: stanfordhealthcare/llama-3.3-70b-instruct
    model_name: meta/llama-3.3-70b-instruct
    tokenizer_name: meta/llama-3.3-70b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/llama3370b/v1"

  - name: stanfordhealthcare/llama-4-scout-17b-16e-instruct
    model_name: meta/llama-4-scout-17b-16e-instruct
    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
    max_sequence_length: 327680
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/llama4-scout/v1"
  
  - name: stanfordhealthcare/llama-4-maverick-17b-128e-instruct-fp8
    model_name: meta/llama-4-maverick-17b-128e-instruct-fp8
    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
    max_sequence_length: 524288
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/llama4-maverick/v1"

  - name: stanfordhealthcare/phi-3.5-mini-instruct
    model_name: microsoft/phi-3.5-mini-instruct
    tokenizer_name: microsoft/phi-3.5-mini-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/phi35mi/v1"

  - name: stanfordhealthcare_shc/gpt-4o-2024-05-13
    model_name: openai/gpt-4o-2024-05-13
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
      deployment: gpt-4o

  - name: stanfordhealthcare_shc/gpt-4o-mini-2024-07-18
    model_name: openai/gpt-4o-mini-2024-07-18
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
      deployment: gpt-4o-mini
  
  - name: stanfordhealthcare_shc/gpt-4-turbo-2024-04-09
    model_name: openai/gpt-4-turbo-2024-04-09
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
      deployment: gpt-4-turbo-2024-04-09

  - name: stanfordhealthcare/claude-3-5-sonnet-20241022
    model_name: anthropic/claude-3-5-sonnet-20241022
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: anthropic.claude-3-5-sonnet-20241022-v2:0
        deployment: Claude35Sonnetv2/awssig4fa

  - name: stanfordhealthcare/claude-3-7-sonnet-20250219
    model_name: anthropic/claude-3-7-sonnet-20250219
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
      args:
        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0
        deployment: awssig4claude37/aswsig4claude37

  - name: stanfordhealthcare/gemini-1.5-pro-001
    model_name: google/gemini-1.5-pro-001
    tokenizer_name: google/gemma-2b
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
      args:
        deployment: gcpgemini/apim-gcp-oauth-fa

  - name: stanfordhealthcare/gpt-4o-mini-2024-07-18
    model_name: openai/gpt-4o-mini-2024-07-18
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4o-mini
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4o-2024-05-13
    model_name: openai/gpt-4o-2024-05-13
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4o
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4-0613
    model_name: openai/gpt-4-0613
    tokenizer_name: openai/o200k_base
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4-turbo-2024-04-09
    model_name: openai/gpt-4-turbo-2024-04-09
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4-turbo
        api_version: 2023-05-15

  - name: stanfordhealthcare/gpt-4.1-2025-04-14
    model_name: openai/gpt-4.1-2025-04-14
    tokenizer_name: openai/o200k_base
    max_sequence_length: 1047576
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: gpt-4.1
        api_version: 2025-01-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/o3-mini-2025-01-31
    model_name: openai/o3-mini-2025-01-31
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: o3-mini
        api_version: 2024-12-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/o1-2024-12-17
    model_name: openai/o1-2024-12-17
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
      args:
        openai_model_name: o1
        api_version: 2024-12-01-preview
        base_url: "{endpoint}/openai-eastus2"

  - name: stanfordhealthcare/deepseek-r1
    model_name: deepseek-ai/deepseek-r1
    tokenizer_name: deepseek-ai/deepseek-r1
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        openai_model_name: deepseek-chat
        output_processor: helm.benchmark.metrics.output_processors.remove_deepseek_r1_thinking
        base_url: "{endpoint}/deepseekr1/v1"

  - name: stanfordhealthcare/llama-3.3-70b-instruct
    model_name: meta/llama-3.3-70b-instruct
    tokenizer_name: meta/llama-3.3-70b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/llama3370b/v1"

  - name: stanfordhealthcare/llama-4-scout-17b-16e-instruct
    model_name: meta/llama-4-scout-17b-16e-instruct
    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
    max_sequence_length: 327680
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/llama4-scout/v1"

  - name: stanfordhealthcare/llama-4-maverick-17b-128e-instruct-fp8
    model_name: meta/llama-4-maverick-17b-128e-instruct-fp8
    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
    max_sequence_length: 524288
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/llama4-maverick/v1"

  - name: stanfordhealthcare/phi-3.5-mini-instruct
    model_name: microsoft/phi-3.5-mini-instruct
    tokenizer_name: microsoft/phi-3.5-mini-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
      args:
        base_url: "{endpoint}/phi35mi/v1"

  - name: stanfordhealthcare_shc/gpt-4o-2024-05-13
    model_name: openai/gpt-4o-2024-05-13
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
      deployment: gpt-4o

  - name: stanfordhealthcare_shc/gpt-4o-mini-2024-07-18
    model_name: openai/gpt-4o-mini-2024-07-18
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
      deployment: gpt-4o-mini

  - name: stanfordhealthcare_shc/gpt-4-turbo-2024-04-09
    model_name: openai/gpt-4-turbo-2024-04-09
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
      deployment: gpt-4-turbo-2024-04-09

  # Adobe
  - name: adobe/giga-gan
    model_name: adobe/giga-gan
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.adobe_vision_client.AdobeVisionClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  # AI21 Labs

  - name: ai21/j2-large
    model_name: ai21/j2-large
    tokenizer_name: ai21/j2-tokenizer
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.ai21_client.AI21Client"

  - name: ai21/j2-grande
    model_name: ai21/j2-grande
    tokenizer_name: ai21/j2-tokenizer
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.ai21_client.AI21Client"

  - name: ai21/j2-jumbo
    model_name: ai21/j2-jumbo
    tokenizer_name: ai21/j2-tokenizer
    max_sequence_length: 6000
    client_spec:
      class_name: "helm.clients.ai21_client.AI21Client"

  - name: ai21/jamba-instruct
    model_name: ai21/jamba-instruct
    tokenizer_name: ai21/jamba-instruct-tokenizer
    max_sequence_length: 256000
    client_spec:
      class_name: "helm.clients.ai21_client.AI21ChatClient"

  - name: ai21/jamba-1.5-mini
    model_name: ai21/jamba-1.5-mini
    tokenizer_name: ai21/jamba-1.5-mini-tokenizer
    max_sequence_length: 256000
    client_spec:
      class_name: "helm.clients.ai21_client.AI21ChatClient"

  - name: ai21/jamba-1.5-large
    model_name: ai21/jamba-1.5-large
    tokenizer_name: ai21/jamba-1.5-large-tokenizer
    max_sequence_length: 256000
    client_spec:
      class_name: "helm.clients.ai21_client.AI21ChatClient"

  # Aleph Alpha
  - name: AlephAlpha/luminous-base
    model_name: AlephAlpha/luminous-base
    tokenizer_name: AlephAlpha/luminous-base
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"

  - name: AlephAlpha/luminous-extended
    model_name: AlephAlpha/luminous-extended
    tokenizer_name: AlephAlpha/luminous-extended
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"

  - name: AlephAlpha/luminous-supreme
    model_name: AlephAlpha/luminous-supreme
    tokenizer_name: AlephAlpha/luminous-supreme
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"

  # TODO: Add luminous-world once it is released

  - name: AlephAlpha/m-vader
    model_name: AlephAlpha/m-vader
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.aleph_alpha_image_generation_client.AlephAlphaImageGenerationClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"


  # Amazon nova models
  - name: amazon/nova-premier-v1:0
    model_name: amazon/nova-premier-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockNovaClient"
      args:
        bedrock_model_id: us.amazon.nova-premier-v1:0

  - name: amazon/nova-pro-v1:0
    model_name: amazon/nova-pro-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 300000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockNovaClient"

  - name: amazon/nova-lite-v1:0
    model_name: amazon/nova-lite-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 300000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockNovaClient"

  - name: amazon/nova-micro-v1:0
    model_name: amazon/nova-micro-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockNovaClient"

  # Titan on Amazon Bedrock

  - name: amazon/titan-text-lite-v1
    model_name: amazon/titan-text-lite-v1
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 4000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockTitanClient"

  - name: amazon/titan-text-express-v1
    model_name: amazon/titan-text-express-v1
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 8000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
  
  # Mistral on Amazon Bedrock

  - name: amazon/mistral-7b-instruct-v0:2
    model_name: mistralai/amazon-mistral-7b-instruct-v0:2
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 8000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockMistralClient"
  
  - name: amazon/mixtral-8x7b-instruct-v0:1
    model_name: mistralai/amazon-mixtral-8x7b-instruct-v0:1
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 4000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockMistralClient"
  
  - name: amazon/mistral-large-2402-v1:0
    model_name: mistralai/amazon-mistral-large-2402-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 8000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockMistralClient"
  
  - name: amazon/mistral-small-2402-v1:0
    model_name: mistralai/amazon-mistral-small-2402-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 8000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockMistralClient"

  - name: amazon/mistral-large-2407-v1:0
    model_name: mistralai/amazon-mistral-large-2407-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 8000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockMistralClient"

  # Llama 3 on Amazon Bedrock
  
  - name: amazon/llama3-8b-instruct-v1:0
    model_name: meta/amazon-llama3-8b-instruct-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 2000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockLlamaClient"

  - name: amazon/llama3-70b-instruct-v1:0
    model_name: meta/amazon-llama3-70b-instruct-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 2000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockLlamaClient"

  - name: amazon/llama3-1-405b-instruct-v1:0
    model_name: meta/amazon-llama3-1-405b-instruct-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 2000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockLlamaClient"

  - name: amazon/llama3-1-70b-instruct-v1:0
    model_name: meta/amazon-llama3-1-70b-instruct-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 2000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockLlamaClient"

  
  - name: amazon/llama3-1-8b-instruct-v1:0
    model_name: meta/amazon-llama3-1-8b-instruct-v1:0
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 2000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockLlamaClient"

  # Anthropic
  - name: anthropic/claude-v1.3
    model_name: anthropic/claude-v1.3
    tokenizer_name: anthropic/claude
    max_sequence_length: 8000
    max_sequence_and_generated_tokens_length: 9016
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicClient"

  - name: anthropic/claude-instant-v1
    model_name: anthropic/claude-instant-v1
    tokenizer_name: anthropic/claude
    max_sequence_length: 8000
    max_sequence_and_generated_tokens_length: 9016
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicClient"

  - name: anthropic/claude-instant-1.2
    model_name: anthropic/claude-instant-1.2
    tokenizer_name: anthropic/claude
    max_sequence_length: 8000
    max_sequence_and_generated_tokens_length: 9016
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicClient"

  - name: anthropic/claude-2.0
    model_name: anthropic/claude-2.0
    tokenizer_name: anthropic/claude
    max_sequence_length: 8000
    max_sequence_and_generated_tokens_length: 9016
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicClient"

  - name: anthropic/claude-2.1
    model_name: anthropic/claude-2.1
    tokenizer_name: anthropic/claude
    max_sequence_length: 8000
    max_sequence_and_generated_tokens_length: 9016
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicClient"

  - name: anthropic/claude-3-sonnet-20240229
    model_name: anthropic/claude-3-sonnet-20240229
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-haiku-20240307
    model_name: anthropic/claude-3-haiku-20240307
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-opus-20240229
    model_name: anthropic/claude-3-opus-20240229
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-5-haiku-20241022
    model_name: anthropic/claude-3-5-haiku-20241022
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-5-sonnet-20240620
    model_name: anthropic/claude-3-5-sonnet-20240620
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-5-sonnet-20241022
    model_name: anthropic/claude-3-5-sonnet-20241022
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-7-sonnet-20250219
    model_name: anthropic/claude-3-7-sonnet-20250219
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-3-7-sonnet-20250219-thinking-10k
    model_name: anthropic/claude-3-7-sonnet-20250219-thinking-10k
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
      args:
        anthropic_model_name: claude-3-7-sonnet-20250219
        thinking_budget_tokens: 10000
        stream: true

  - name: anthropic/claude-sonnet-4-20250514
    model_name: anthropic/claude-sonnet-4-20250514
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-sonnet-4-20250514-thinking-10k
    model_name: anthropic/claude-sonnet-4-20250514-thinking-10k
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
      args:
        anthropic_model_name: claude-sonnet-4-20250514
        thinking_budget_tokens: 10000
        stream: true

  - name: anthropic/claude-opus-4-20250514
    model_name: anthropic/claude-opus-4-20250514
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-opus-4-20250514-thinking-10k
    model_name: anthropic/claude-opus-4-20250514-thinking-10k
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
      args:
        anthropic_model_name: claude-opus-4-20250514
        thinking_budget_tokens: 10000
        stream: true

  - name: anthropic/claude-sonnet-4-5-20250929
    model_name: anthropic/claude-sonnet-4-5-20250929
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-haiku-4-5-20251001
    model_name: anthropic/claude-haiku-4-5-20251001
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-sonnet-4-6
    model_name: anthropic/claude-sonnet-4-6
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/claude-opus-4-6
    model_name: anthropic/claude-opus-4-6
    tokenizer_name: anthropic/claude
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"

  - name: anthropic/stanford-online-all-v4-s3
    deprecated: true # Closed model, not accessible via API
    model_name: anthropic/stanford-online-all-v4-s3
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.anthropic_client.AnthropicLegacyClient"

  # Cohere
  - name: cohere/command
    model_name: cohere/command
    tokenizer_name: cohere/command
    max_sequence_length: 2019 # TODO: verify this
    max_request_length: 2020 # TODO: verify this
    client_spec:
      class_name: "helm.clients.cohere_client.CohereClient"

  - name: cohere/command-light
    model_name: cohere/command-light
    tokenizer_name: cohere/command-light
    max_sequence_length: 2019 # TODO: verify this
    max_request_length: 2020 # TODO: verify this
    client_spec:
      class_name: "helm.clients.cohere_client.CohereClient"

  - name: cohere/command-r
    model_name: cohere/command-r
    tokenizer_name: cohere/command-r
    max_sequence_length: 128000
    max_request_length: 128000
    client_spec:
      class_name: "helm.clients.cohere_client.CohereChatClient"

  - name: cohere/command-r-plus
    model_name: cohere/command-r-plus
    tokenizer_name: cohere/command-r-plus
    # "We have a known issue where prompts between 112K - 128K in length
    # result in bad generations."
    # Source: https://docs.cohere.com/docs/command-r-plus
    max_sequence_length: 110000
    max_request_length: 110000
    client_spec:
      class_name: "helm.clients.cohere_client.CohereChatClient"

  - name: huggingface/command-a-03-2025
    model_name: cohere/command-a-03-2025
    tokenizer_name: cohere/command-a-03-2025
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: CohereLabs/c4ai-command-a-03-2025
        device_map: auto

  # Craiyon

  - name: craiyon/dalle-mini
    model_name: craiyon/dalle-mini
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.dalle_mini_client.DALLEMiniClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: craiyon/dalle-mega
    model_name: craiyon/dalle-mega
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.dalle_mini_client.DALLEMiniClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  # Databricks

  - name: together/dbrx-instruct
    model_name: databricks/dbrx-instruct
    tokenizer_name: databricks/dbrx-instruct
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  # DeepFloyd

  - name: DeepFloyd/IF-I-M-v1.0
    model_name: DeepFloyd/IF-I-M-v1.0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: DeepFloyd/IF-I-L-v1.0
    model_name: DeepFloyd/IF-I-L-v1.0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: DeepFloyd/IF-I-XL-v1.0
    model_name: DeepFloyd/IF-I-XL-v1.0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  # Deepseek

  - name: together/deepseek-llm-67b-chat
    model_name: deepseek-ai/deepseek-llm-67b-chat
    tokenizer_name: deepseek-ai/deepseek-llm-67b-chat
    max_sequence_length: 4095
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/deepseek-v3
    model_name: deepseek-ai/deepseek-v3
    tokenizer_name: deepseek-ai/deepseek-v3
    max_sequence_length: 16384
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        disable_logprobs: True

  - name: together/deepseek-v3.1
    model_name: deepseek-ai/deepseek-v3.1
    tokenizer_name: deepseek-ai/deepseek-v3.1
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        # The model name is case sensitive on Together AI for this model
        together_model: deepseek-ai/DeepSeek-V3.1

  - name: together/deepseek-r1-0528
    model_name: deepseek-ai/deepseek-r1-0528
    tokenizer_name: deepseek-ai/deepseek-r1
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: deepseek-ai/deepseek-r1
        parse_thinking: true
        disable_logprobs: True

  - name: together/deepseek-r1-distill-llama-70b
    model_name: deepseek-ai/deepseek-r1-distill-llama-70b
    tokenizer_name: deepseek-ai/deepseek-r1-distill-llama-70b
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/deepseek-r1-distill-qwen-14b
    model_name: deepseek-ai/deepseek-r1-distill-qwen-14b
    tokenizer_name: deepseek-ai/deepseek-r1-distill-qwen-14b
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  # Gooseai

  # TODO: Migrate these models to use OpenAIClient

  ## EleutherAI
  # - name: gooseai/gpt-neo-20b
  #   model_name: eleutherai/gpt-neox-20b
  #   tokenizer_name: EleutherAI/gpt-neox-20b
  #   max_sequence_length: 2048
  #   max_request_length: 2049
  #   client_spec:
  #     class_name: "helm.clients.goose_ai_client.GooseAIClient"

  # - name: gooseai/gpt-j-6b
  #   model_name: eleutherai/gpt-j-6b
  #   tokenizer_name: EleutherAI/gpt-j-6B
  #   max_sequence_length: 2048
  #   max_request_length: 2049
  #   client_spec:
  #     class_name: "helm.clients.goose_ai_client.GooseAIClient"

  # Google
  # See: https://cloud.google.com/vertex-ai/docs/generative-ai/learn/model-versioning

  ## Gemini
  # See: https://ai.google.dev/models/gemini#model_variations
  - name: google/gemini-pro
    model_name: google/gemini-pro
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 30720
    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.0-pro-001
    model_name: google/gemini-1.0-pro-001
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 30720
    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.0-pro-002
    model_name: google/gemini-1.0-pro-002
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 30720
    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-pro-vision
    model_name: google/gemini-pro-vision
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 12288
    max_sequence_and_generated_tokens_length: 16384 # Officially max_sequence_length + 4096, in practice max_output_tokens <= 2048 for vision models
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.0-pro-vision-001
    model_name: google/gemini-1.0-pro-vision-001
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 12288
    max_sequence_and_generated_tokens_length: 16384
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.5-flash-001
    model_name: google/gemini-1.5-flash-001
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.5-pro-001
    model_name: google/gemini-1.5-pro-001
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.5-pro-preview-0409
    model_name: google/gemini-1.5-pro-preview-0409
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.5-pro-preview-0514
    model_name: google/gemini-1.5-pro-preview-0514
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.5-flash-preview-0514
    model_name: google/gemini-1.5-flash-preview-0514
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  ## Gemini with different safety settings
  - name: google/gemini-1.5-pro-001-safety-default
    model_name: google/gemini-1.5-pro-001-safety-default
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: gemini-1.5-pro-001
        safety_settings_preset: default

  - name: google/gemini-1.5-pro-001-safety-block-none
    model_name: google/gemini-1.5-pro-001-safety-block-none
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: gemini-1.5-pro-001
        safety_settings_preset: block_none

  - name: google/gemini-1.5-flash-001-safety-default
    model_name: google/gemini-1.5-flash-001-safety-default
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: gemini-1.5-flash-001
        safety_settings_preset: default

  - name: google/gemini-1.5-flash-001-safety-block-none
    model_name: google/gemini-1.5-flash-001-safety-block-none
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: gemini-1.5-flash-001
        safety_settings_preset: block_none

  - name: google/gemini-1.5-pro-002
    model_name: google/gemini-1.5-pro-002
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-1.5-flash-002
    model_name: google/gemini-1.5-flash-002
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.0-pro-exp-02-05
    model_name: google/gemini-2.0-pro-exp-02-05
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.0-flash-exp
    model_name: google/gemini-2.0-flash-exp
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.0-flash-001
    model_name: google/gemini-2.0-flash-001
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.0-flash-lite-preview-02-05
    model_name: google/gemini-2.0-flash-lite-preview-02-05
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.0-flash-lite-001
    model_name: google/gemini-2.0-flash-lite-001
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.0-flash-thinking-exp-01-21
    model_name: google/gemini-2.0-flash-thinking-exp-01-21
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/gemini-2.5-flash-lite-thinking-disabled
    model_name: google/gemini-2.5-flash-lite-thinking-disabled
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        genai_model: gemini-2.5-flash-lite
        thinking_config:
          thinking_budget: 0

  - name: google/gemini-2.5-flash-thinking-disabled
    model_name: google/gemini-2.5-flash-thinking-disabled
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        genai_model: gemini-2.5-flash
        thinking_config:
          thinking_budget: 0

  - name: google/gemini-2.5-flash-lite
    model_name: google/gemini-2.5-flash-lite
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"

  - name: google/gemini-2.5-flash
    model_name: google/gemini-2.5-flash
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        thinking_config:
          include_thoughts: true

  - name: google/gemini-2.5-pro
    model_name: google/gemini-2.5-pro
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576   # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        thinking_config:
          include_thoughts: true

  - name: google/gemini-3-pro-preview
    model_name: google/gemini-3-pro-preview
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        thinking_config:
          include_thoughts: true

  - name: google/gemini-3.1-pro-preview
    model_name: google/gemini-3.1-pro-preview
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-1-pro
    # TODO: Max output tokens: 65536
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        thinking_config:
          include_thoughts: true

  - name: google/gemini-robotics-er-1.5-preview
    model_name: google/gemini-robotics-er-1.5-preview
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1048576  # Source: https://deepmind.google/models/gemini-robotics/gemini-robotics-er/
    client_spec:
      class_name: "helm.clients.google_genai_client.GoogleGenAIClient"
      args:
        genai_use_vertexai: False

  - name: google/gemini-1.5-flash-8b-001
    model_name: google/gemini-1.5-flash-8b-001
    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
    # TODO: Max output tokens: 8192
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"

  - name: google/llama-3.1-8b-instruct
    model_name: meta/llama-3.1-8b-instruct
    tokenizer_name: meta/llama-3.1-8b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: publishers/meta/models/llama-3.1-8b-instruct-maas

  - name: google/llama-3.1-70b-instruct
    model_name: meta/llama-3.1-70b-instruct
    tokenizer_name: meta/llama-3.1-8b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: publishers/meta/models/llama-3.1-70b-instruct-maas

  - name: google/llama-3.1-405b-instruct
    model_name: meta/llama-3.1-405b-instruct
    tokenizer_name: meta/llama-3.1-8b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
      args:
        vertexai_model: publishers/meta/models/llama-3.1-405b-instruct-maas

  ## Gemma
  - name: together/gemma-2b
    model_name: google/gemma-2b
    tokenizer_name: google/gemma-2b
    max_sequence_length: 7167
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/gemma-2b-it
    model_name: google/gemma-2b-it
    tokenizer_name: google/gemma-2b
    max_sequence_length: 7167
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/gemma-7b
    model_name: google/gemma-7b
    tokenizer_name: google/gemma-2b
    max_sequence_length: 7167
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/gemma-7b-it
    model_name: google/gemma-7b-it
    tokenizer_name: google/gemma-2b
    max_sequence_length: 7167
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/gemma-2-9b-it
    model_name: google/gemma-2-9b-it
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/gemma-2-27b-it
    model_name: google/gemma-2-27b-it
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  ## MedLM
  - name: google/medlm-medium
    model_name: google/medlm-medium
    tokenizer_name: google/text-bison@001
    max_sequence_length: 6000 # Officially 8192
    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: google/medlm-large
    model_name: google/medlm-large
    tokenizer_name: google/text-bison@001
    max_sequence_length: 6000 # Officially 8192
    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  ## PaliGemma
  - name: google/paligemma-3b-mix-224
    model_name: google/paligemma-3b-mix-224
    tokenizer_name: google/gemma-2b
    max_sequence_length: 7167
    client_spec:
      class_name: "helm.clients.vision_language.paligemma_client.PaliGemmaClient"

  - name: google/paligemma-3b-mix-448
    model_name: google/paligemma-3b-mix-448
    tokenizer_name: google/gemma-2b
    max_sequence_length: 7167
    client_spec:
      class_name: "helm.clients.vision_language.paligemma_client.PaliGemmaClient"

  ## PaLM 2
  - name: google/text-bison@001
    model_name: google/text-bison@001
    tokenizer_name: google/text-bison@001
    max_sequence_length: 6000 # Officially 8192
    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: google/text-bison@002
    model_name: google/text-bison@002
    tokenizer_name: google/text-bison@002
    max_sequence_length: 6000 # Officially 8192
    max_sequence_and_generated_tokens_length: 9216
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: google/text-bison-32k
    model_name: google/text-bison-32k
    tokenizer_name: google/text-bison@001
    max_sequence_length: 32000
    max_sequence_and_generated_tokens_length: 32000
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"


  - name: google/text-unicorn@001
    model_name: google/text-unicorn@001
    tokenizer_name: google/text-unicorn@001
    max_sequence_length: 6000 # Officially 8192
    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: google/code-bison@001
    model_name: google/code-bison@001
    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
    max_sequence_length: 6000 # Officially 6144
    max_sequence_and_generated_tokens_length: 7000 # Officially 7168
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: google/code-bison@002
    model_name: google/code-bison@002
    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
    max_sequence_length: 6000 # Officially 6144
    max_sequence_and_generated_tokens_length: 7168
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: google/code-bison-32k
    model_name: google/code-bison-32k
    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
    max_sequence_length: 32000
    max_sequence_and_generated_tokens_length: 32000
    client_spec:
      class_name: "helm.clients.vertexai_client.VertexAITextClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  # HuggingFace

  ## AI Singapore
  - name: huggingface/sea-lion-7b
    model_name: aisingapore/sea-lion-7b
    tokenizer_name: aisingapore/sea-lion-7b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        trust_remote_code: true

  - name: huggingface/sea-lion-7b-instruct
    model_name: aisingapore/sea-lion-7b-instruct
    tokenizer_name: aisingapore/sea-lion-7b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        trust_remote_code: true

  - name: huggingface/llama3-8b-cpt-sea-lionv2-base
    model_name: aisingapore/llama3-8b-cpt-sea-lionv2-base
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/llama3-8b-cpt-sea-lionv2.1-instruct
    model_name: aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/gemma2-9b-cpt-sea-lionv3-base
    model_name: aisingapore/gemma2-9b-cpt-sea-lionv3-base
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/gemma2-9b-cpt-sea-lionv3-instruct
    model_name: aisingapore/gemma2-9b-cpt-sea-lionv3-instruct
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/llama3.1-8b-cpt-sea-lionv3-base
    model_name: aisingapore/llama3.1-8b-cpt-sea-lionv3-base
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/llama3.1-8b-cpt-sea-lionv3-instruct
    model_name: aisingapore/llama3.1-8b-cpt-sea-lionv3-instruct
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/llama3.1-70b-cpt-sea-lionv3-base
    model_name: aisingapore/llama3.1-70b-cpt-sea-lionv3-base
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/llama3.1-70b-cpt-sea-lionv3-instruct
    model_name: aisingapore/llama3.1-70b-cpt-sea-lionv3-instruct
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  ## Bigcode
  - name: huggingface/santacoder
    model_name: bigcode/santacoder
    tokenizer_name: bigcode/santacoder
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/starcoder
    model_name: bigcode/starcoder
    tokenizer_name: bigcode/starcoder
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  ## Biomistral 

  - name: huggingface/biomistral-7b
    model_name: biomistral/biomistral-7b
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  ## Databricks
  - name: huggingface/dolly-v2-3b
    model_name: databricks/dolly-v2-3b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/dolly-v2-7b
    model_name: databricks/dolly-v2-7b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/dolly-v2-12b
    model_name: databricks/dolly-v2-12b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  ## EleutherAI
  - name: huggingface/pythia-1b-v0
    model_name: eleutherai/pythia-1b-v0
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/pythia-2.8b-v0
    model_name: eleutherai/pythia-2.8b-v0
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/pythia-6.9b
    model_name: eleutherai/pythia-6.9b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/pythia-12b-v0
    model_name: eleutherai/pythia-12b-v0
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/gpt-j-6b
    model_name: eleutherai/gpt-j-6b
    tokenizer_name: EleutherAI/gpt-j-6B
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/gpt-neox-20b
    model_name: eleutherai/gpt-neox-20b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  ## Google
  - name: huggingface/gemma-2-9b
    model_name: google/gemma-2-9b
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/gemma-2-9b-it
    model_name: google/gemma-2-9b-it
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/gemma-2-27b
    model_name: google/gemma-2-27b
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16

  - name: huggingface/gemma-2-27b-it
    model_name: google/gemma-2-27b-it
    tokenizer_name: google/gemma-2-9b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: torch.bfloat16
      
  ## Google
  - name: huggingface/medgemma-4b-it
    model_name: google/medgemma-4b-it
    tokenizer_name: google/medgemma-4b-it
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        trust_remote_code: true
        torch_dtype: torch.bfloat16

  ## LMSYS
  - name: huggingface/vicuna-7b-v1.3
    model_name: lmsys/vicuna-7b-v1.3
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/vicuna-13b-v1.3
    model_name: lmsys/vicuna-13b-v1.3
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  ## Meditron 

  - name: huggingface/meditron-7b
    model_name: epfl-llm/meditron-7b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4094
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  ## Meta
  - name: huggingface/llama-3.1-8b-instruct
    model_name: meta/llama-3.1-8b-instruct
    tokenizer_name: meta/llama-3.1-8b-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: meta-llama/Llama-3.1-8B-Instruct

  - name: huggingface/llama-3.2-1b-instruct
    model_name: meta/llama-3.2-1b-instruct
    tokenizer_name: meta/llama-3.2-1b-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct

  - name: huggingface/opt-175b
    model_name: meta/opt-175b
    tokenizer_name: facebook/opt-66b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: facebook/opt-175b

  - name: huggingface/opt-66b
    model_name: meta/opt-66b
    tokenizer_name: facebook/opt-66b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: facebook/opt-66b

  - name: huggingface/opt-6.7b
    model_name: meta/opt-6.7b
    tokenizer_name: facebook/opt-66b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: facebook/opt-6.7b

  - name: huggingface/opt-1.3b
    model_name: meta/opt-1.3b
    tokenizer_name: facebook/opt-66b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: facebook/opt-1.3b

  ## Microsoft
  - name: huggingface/llava-1.5-7b-hf
    model_name: microsoft/llava-1.5-7b-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
  
  - name: huggingface/llava-1.5-13b-hf
    model_name: microsoft/llava-1.5-13b-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"

  - name: huggingface/llava-v1.6-vicuna-7b-hf
    model_name: uw-madison/llava-v1.6-vicuna-7b-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"

  - name: huggingface/llava-v1.6-vicuna-13b-hf
    model_name: uw-madison/llava-v1.6-vicuna-13b-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"

  - name: huggingface/llava-v1.6-mistral-7b-hf
    model_name: uw-madison/llava-v1.6-mistral-7b-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"

  - name: huggingface/llava-v1.6-34b-hf
    model_name: uw-madison/llava-v1.6-34b-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
      
  ## NECTEC
  - name: huggingface/Pathumma-llm-text-1.0.0
    model_name: nectec/Pathumma-llm-text-1.0.0
    tokenizer_name: nectec/Pathumma-llm-text-1.0.0
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/OpenThaiLLM-Prebuilt-7B
    model_name: nectec/OpenThaiLLM-Prebuilt-7B
    tokenizer_name: nectec/OpenThaiLLM-Prebuilt-7B
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        apply_chat_template: false
    
  ## KAIST AI
  - name: huggingface/prometheus-vision-13b-v1.0-hf
    model_name: kaistai/prometheus-vision-13b-v1.0-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"

  ## OpenFlamingo
  - name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
    model_name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
    tokenizer_name: anas-awadalla/mpt-7b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.open_flamingo_client.OpenFlamingoClient"
      args:
        checkpoint_path: "openflamingo/OpenFlamingo-9B-vitl-mpt7b"
        tokenizer_name: "anas-awadalla-2/mpt-7b"
        cross_attn_every_n_layers: 4

  ## Marin Community
  - name: huggingface/marin-8b-instruct
    model_name: marin-community/marin-8b-instruct
    tokenizer_name: marin-community/marin-8b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: together/marin-8b-instruct
    model_name: marin-community/marin-8b-instruct
    tokenizer_name: marin-community/marin-8b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient" 

  ## Microsoft
  - name: together/phi-2
    model_name: microsoft/phi-2
    tokenizer_name: microsoft/phi-2
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient" 

  - name: huggingface/phi-3-small-8k-instruct
    model_name: microsoft/phi-3-small-8k-instruct
    tokenizer_name: microsoft/phi-3-small-8k-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        torch_dtype: auto
        trust_remote_code: true

  - name: huggingface/phi-3-medium-4k-instruct
    model_name: microsoft/phi-3-medium-4k-instruct
    tokenizer_name: microsoft/phi-3-medium-4k-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: auto

  - name: huggingface/phi-3.5-mini-instruct-4bit
    model_name: microsoft/phi-3.5-mini-instruct
    tokenizer_name: microsoft/phi-3.5-mini-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: "float16"
        quantization_config:
          load_in_4bit: true
        attn_implementation: "flash_attention_2"
  
  - name: huggingface/phi-3.5-mini-instruct
    model_name: microsoft/phi-3.5-mini-instruct
    tokenizer_name: microsoft/phi-3.5-mini-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/phi-3.5-moe-instruct
    model_name: microsoft/phi-3.5-moe-instruct
    tokenizer_name: microsoft/phi-3.5-mini-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        torch_dtype: auto

  ## Mistral AI
  - name: huggingface/bakLlava-v1-hf
    model_name: mistralai/bakLlava-v1-hf
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"

  ## Moonshot AI
  - name: together/kimi-k2-instruct
    model_name: moonshotai/kimi-k2-instruct
    tokenizer_name: moonshotai/kimi-k2-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/kimi-k2-instruct-0905
    model_name: moonshotai/kimi-k2-instruct-0905
    tokenizer_name: moonshotai/kimi-k2-instruct
    max_sequence_length: 262144
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/kimi-k2-thinking
    model_name: moonshotai/kimi-k2-thinking
    tokenizer_name: moonshotai/kimi-k2-thinking
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  ## MosaicML
  - name: huggingface/mpt-7b
    model_name: mosaicml/mpt-7b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: mosaicml/mpt-7b

  - name: huggingface/mpt-instruct-7b
    model_name: mosaicml/mpt-instruct-7b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: mosaicml/mpt-7b-instruct

  - name: huggingface/mpt-30b
    model_name: mosaicml/mpt-30b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/mpt-instruct-30b
    model_name: mosaicml/mpt-instruct-30b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: mosaicml/mpt-30b-instruct

  ## OpenAI
  - name: huggingface/gpt2
    model_name: openai/gpt2
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 1024
    max_request_length: 1025
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: openai-community/gpt2

  ## OpenThaiGPT
  - name: huggingface/openthaigpt-1.0.0-7b-chat
    model_name: openthaigpt/openthaigpt-1.0.0-7b-chat
    tokenizer_name: openthaigpt/openthaigpt-1.0.0-7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/openthaigpt-1.0.0-13b-chat
    model_name: openthaigpt/openthaigpt-1.0.0-13b-chat
    tokenizer_name: openthaigpt/openthaigpt-1.0.0-7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/openthaigpt-1.0.0-70b-chat
    model_name: openthaigpt/openthaigpt-1.0.0-70b-chat
    tokenizer_name: huggingface/openthaigpt-1.0.0-7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  ## SAIL (SEA AI Lab)
  - name: huggingface/sailor-7b
    model_name: sail/sailor-7b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        apply_chat_template: false

  - name: huggingface/sailor-7b-chat
    model_name: sail/sailor-7b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/sailor-14b
    model_name: sail/sailor-14b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        apply_chat_template: false

  - name: huggingface/sailor-14b-chat
    model_name: sail/sailor-14b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  # SambaNova
  - name: huggingface/sambalingo-thai-base
    model_name: sambanova/sambalingo-thai-base
    tokenizer_name: sambanova/sambalingo-thai-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base

  - name: huggingface/sambalingo-thai-chat
    model_name: sambanova/sambalingo-thai-chat
    tokenizer_name: sambanova/sambalingo-thai-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base

  - name: huggingface/sambalingo-thai-base-70b
    model_name: sambanova/sambalingo-thai-base-70b
    tokenizer_name: sambanova/sambalingo-thai-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base-70B
        device_map: auto

  - name: huggingface/sambalingo-thai-chat-70b
    model_name: sambanova/sambalingo-thai-chat-70b
    tokenizer_name: sambanova/sambalingo-thai-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base-70B
        device_map: auto

  ## SCB10X
  - name: huggingface/typhoon-7b
    model_name: scb10x/typhoon-7b
    tokenizer_name: scb10x/typhoon-7b
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/typhoon-v1.5-8b
    model_name: scb10x/typhoon-v1.5-8b
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/typhoon-v1.5-8b-instruct
    model_name: scb10x/typhoon-v1.5-8b-instruct
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/typhoon-v1.5-72b
    model_name: scb10x/typhoon-v1.5-72b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/typhoon-v1.5-72b-instruct
    model_name: scb10x/typhoon-v1.5-72b-instruct
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/llama-3-typhoon-v1.5x-8b-instruct
    model_name: scb10x/llama-3-typhoon-v1.5x-8b-instruct
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/llama-3-typhoon-v1.5x-70b-instruct
    model_name: scb10x/llama-3-typhoon-v1.5x-70b-instruct
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  # Alibaba DAMO Academy
  - name: huggingface/seallm-7b-v2
    model_name: damo/seallm-7b-v2
    tokenizer_name: damo/seallm-7b-v2
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2

  - name: huggingface/seallm-7b-v2.5
    model_name: damo/seallm-7b-v2.5
    tokenizer_name: damo/seallm-7b-v2.5
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2.5

  ## StabilityAI
  - name: huggingface/stablelm-base-alpha-3b
    model_name: stabilityai/stablelm-base-alpha-3b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/stablelm-base-alpha-7b
    model_name: stabilityai/stablelm-base-alpha-7b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  # Upstage
  - name: huggingface/solar-pro-preview-instruct
    model_name: upstage/solar-pro-preview-instruct
    tokenizer_name: upstage/solar-pro-preview-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        torch_dtype: auto
        trust_remote_code: true

  ## Text-to-Image Diffusion Models

  - name: huggingface/dreamlike-diffusion-v1-0
    model_name: huggingface/dreamlike-diffusion-v1-0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/dreamlike-photoreal-v2-0
    model_name: huggingface/dreamlike-photoreal-v2-0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/openjourney-v1-0
    model_name: huggingface/openjourney-v1-0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/openjourney-v2-0
    model_name: huggingface/openjourney-v2-0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/redshift-diffusion
    model_name: huggingface/redshift-diffusion
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/promptist-stable-diffusion-v1-4
    model_name: huggingface/promptist-stable-diffusion-v1-4
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-v1-4
    model_name: huggingface/stable-diffusion-v1-4
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-v1-5
    model_name: huggingface/stable-diffusion-v1-5
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-v2-base
    model_name: huggingface/stable-diffusion-v2-base
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-v2-1-base
    model_name: huggingface/stable-diffusion-v2-1-base
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-safe-weak
    model_name: huggingface/stable-diffusion-safe-weak
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-safe-medium
    model_name: huggingface/stable-diffusion-safe-medium
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-safe-strong
    model_name: huggingface/stable-diffusion-safe-strong
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/stable-diffusion-safe-max
    model_name: huggingface/stable-diffusion-safe-max
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: huggingface/vintedois-diffusion-v0-1
    model_name: huggingface/vintedois-diffusion-v0-1
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: segmind/Segmind-Vega
    model_name: segmind/Segmind-Vega
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: segmind/SSD-1B
    model_name: segmind/SSD-1B
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  - name: stabilityai/stable-diffusion-xl-base-1.0
    model_name: stabilityai/stable-diffusion-xl-base-1.0
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  # HuggingFaceM4
  - name: HuggingFaceM4/idefics2-8b
    model_name: HuggingFaceM4/idefics2-8b
    # From https://huggingface.co/docs/transformers/main/en/model_doc/idefics2,
    # "constructs a IDEFICS2 processor which wraps a LLama tokenizer."
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.huggingface_vision2seq_client.HuggingFaceVision2SeqClient"

  - name: HuggingFaceM4/idefics-9b
    model_name: HuggingFaceM4/idefics-9b
    tokenizer_name: HuggingFaceM4/idefics-9b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"

  - name: HuggingFaceM4/idefics-9b-instruct
    model_name: HuggingFaceM4/idefics-9b-instruct
    tokenizer_name: HuggingFaceM4/idefics-9b-instruct
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"

  - name: HuggingFaceM4/idefics-80b
    model_name: HuggingFaceM4/idefics-80b
    tokenizer_name: HuggingFaceM4/idefics-80b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"

  - name: HuggingFaceM4/idefics-80b-instruct
    model_name: HuggingFaceM4/idefics-80b-instruct
    tokenizer_name: HuggingFaceM4/idefics-80b-instruct
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"

  # Lexica
  - name: lexica/search-stable-diffusion-1.5
    model_name: lexica/search-stable-diffusion-1.5
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 200
    client_spec:
      class_name: "helm.clients.image_generation.lexica_client.LexicaClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.lexica_search_window_service.LexicaSearchWindowService"

  # Kakao
  - name: kakaobrain/mindall-e
    model_name: kakaobrain/mindall-e
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.mindalle_client.MinDALLEClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  # Lighting AI
  - name: lightningai/lit-gpt
    model_name: lightningai/lit-gpt
    tokenizer_name: lightningai/lit-gpt
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.lit_gpt_client.LitGPTClient"
      args:
        checkpoint_dir: "" # Path to the checkpoint directory
        precision: bf16-true

  # Mistral AI

  - name: mistralai/ministral-3b-2410
    model_name: mistralai/ministral-3b-2410
    tokenizer_name: mistralai/Ministral-8B-Instruct-2410
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/ministral-8b-2410
    model_name: mistralai/ministral-8b-2410
    tokenizer_name: mistralai/Ministral-8B-Instruct-2410
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-small-2402
    model_name: mistralai/mistral-small-2402
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-small-2409
    model_name: mistralai/mistral-small-2409
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-small-2501
    model_name: mistralai/mistral-small-2501
    tokenizer_name: mistralai/Mistral-Small-24B-Instruct-2501
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-small-2503
    model_name: mistralai/mistral-small-2503
    tokenizer_name: mistralai/Mistral-Small-24B-Instruct-2501
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-medium-2312
    model_name: mistralai/mistral-medium-2312
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-medium-2505
    model_name: mistralai/mistral-medium-2505
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-large-2402
    model_name: mistralai/mistral-large-2402
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-large-2407
    model_name: mistralai/mistral-large-2407
    tokenizer_name: mistralai/Mistral-Large-Instruct-2407
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/mistral-large-2411
    model_name: mistralai/mistral-large-2411
    tokenizer_name: mistralai/Mistral-Large-Instruct-2411
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/open-mistral-nemo-2407
    model_name: mistralai/open-mistral-nemo-2407
    tokenizer_name: mistralai/Mistral-Nemo-Base-2407
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/pixtral-12b-2409
    model_name: mistralai/pixtral-12b-2409
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"

  - name: mistralai/pixtral-large-2411
    model_name: mistralai/pixtral-large-2411
    tokenizer_name: mistralai/Mistral-Large-Instruct-2407
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.mistral_client.MistralAIClient"


  # Neurips
  - name: neurips/local
    model_name: neurips/local
    tokenizer_name: neurips/local
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.http_model_client.HTTPModelClient"

  # Nvidia
  - name: nvidia/megatron-gpt2
    model_name: nvidia/megatron-gpt2
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 1024
    client_spec:
      class_name: "helm.clients.megatron_client.MegatronClient"

  - name: nvidia/nemotron-4-340b-instruct
    model_name: nvidia/nemotron-4-340b-instruct
    tokenizer_name: nvidia/nemotron-4-340b-instruct
    max_sequence_length: 4085
    client_spec:
      class_name: "helm.clients.nvidia_nim_client.NvidiaNimClient"

  # OpenAI

  ## GPT 3 Models

  - name: openai/davinci-002
    model_name: openai/davinci-002
    tokenizer_name: openai/cl100k_base
    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
    # to provide a margin of error.
    max_sequence_length: 16000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAILegacyCompletionsClient"

  - name: openai/babbage-002
    model_name: openai/babbage-002
    tokenizer_name: openai/cl100k_base
    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
    # to provide a margin of error.
    max_sequence_length: 16000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAILegacyCompletionsClient"

  ## GPT 3.5 Turbo Models
  # ChatGPT: https://openai.com/blog/chatgpt

  - name: openai/gpt-3.5-turbo-instruct
    model_name: openai/gpt-3.5-turbo-instruct
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 4096
    max_request_length: 4097
    client_spec:
      class_name: "helm.clients.openai_client.OpenAILegacyCompletionsClient"

  # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
  # sequence length is smaller at 4087 with one user input message and one assistant
  # output message because ChatGPT uses special tokens for message roles and boundaries.
  # We use a rounded-down sequence length of 4000 to account for these special tokens.
  - name: openai/gpt-3.5-turbo-0301
    model_name: openai/gpt-3.5-turbo-0301
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 4000
    max_request_length: 4001
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
  # sequence length is smaller at 4087 with one user input message and one assistant
  # output message because ChatGPT uses special tokens for message roles and boundaries.
  # We use a rounded-down sequence length of 4000 to account for these special tokens.
  - name: openai/gpt-3.5-turbo-0613
    model_name: openai/gpt-3.5-turbo-0613
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 4000
    max_request_length: 4001
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
  # in the openai/gpt-3.5-turbo-0613 comment
  - name: openai/gpt-3.5-turbo-16k-0613
    model_name: openai/gpt-3.5-turbo-16k-0613
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 16000
    max_request_length: 16001
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
  # in the openai/gpt-3.5-turbo-0613 comment
  - name: openai/gpt-3.5-turbo-1106
    model_name: openai/gpt-3.5-turbo-1106
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 16000
    max_request_length: 16001
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
  # in the openai/gpt-3.5-turbo-0613 comment
  - name: openai/gpt-3.5-turbo-0125
    model_name: openai/gpt-3.5-turbo-0125
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 16000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  ## GPT 4 Models

  - name: openai/gpt-4-1106-preview
    model_name: openai/gpt-4-1106-preview
    tokenizer_name: openai/cl100k_base
    # According to https://help.openai.com/en/articles/8555510-gpt-4-turbo,
    # the maximum number of output tokens for this model is 4096
    # TODO: add max_generated_tokens_length of 4096 https://github.com/stanford-crfm/helm/issues/2098
    max_sequence_length: 128000
    max_request_length: 128001
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-0314
    model_name: openai/gpt-4-0314
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 8192
    max_request_length: 8193
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-32k-0314
    model_name: openai/gpt-4-32k-0314
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 32768
    max_request_length: 32769
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-0613
    model_name: openai/gpt-4-0613
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 8192
    max_request_length: 8193
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-32k-0613
    model_name: openai/gpt-4-32k-0613
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 32768
    max_request_length: 32769
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-0125-preview
    model_name: openai/gpt-4-0125-preview
    tokenizer_name: openai/cl100k_base
    # According to https://help.openai.com/en/articles/8555510-gpt-4-turbo,
    # the maximum number of output tokens for this model is 4096
    # TODO: add max_generated_tokens_length of 4096 https://github.com/stanford-crfm/helm/issues/2098
    max_sequence_length: 128000
    max_request_length: 128001
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-turbo-2024-04-09
    model_name: openai/gpt-4-turbo-2024-04-09
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4o-2024-05-13
    model_name: openai/gpt-4o-2024-05-13
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4o-2024-08-06
    model_name: openai/gpt-4o-2024-08-06
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4o-2024-11-20
    model_name: openai/gpt-4o-2024-11-20
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4o-mini-2024-07-18
    model_name: openai/gpt-4o-mini-2024-07-18
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4.1-2025-04-14
    model_name: openai/gpt-4.1-2025-04-14
    tokenizer_name: openai/o200k_base
    max_sequence_length: 1047576
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4.1-mini-2025-04-14
    model_name: openai/gpt-4.1-mini-2025-04-14
    tokenizer_name: openai/o200k_base
    max_sequence_length: 1047576
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4.1-nano-2025-04-14
    model_name: openai/gpt-4.1-nano-2025-04-14
    tokenizer_name: openai/o200k_base
    max_sequence_length: 1047576
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-5-2025-08-07
    model_name: openai/gpt-5-2025-08-07
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"

  - name: openai/gpt-5-mini-2025-08-07
    model_name: openai/gpt-5-mini-2025-08-07
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"

  - name: openai/gpt-5-nano-2025-08-07
    model_name: openai/gpt-5-nano-2025-08-07
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"

  - name: openai/gpt-5.2-2025-12-11
    model_name: openai/gpt-5.2-2025-12-11
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"

  - name: openai/gpt-5.1-2025-11-13
    model_name: openai/gpt-5.1-2025-11-13
    tokenizer_name: openai/o200k_base
    max_sequence_length: 400000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"

  - name: openai/whisper-1_gpt-4o-2024-11-20
    model_name: openai/whisper-1_gpt-4o-2024-11-20
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"

  - name: openai/gpt-4o-transcribe_gpt-4o-2024-11-20
    model_name: openai/gpt-4o-transcribe_gpt-4o-2024-11-20
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"

  - name: openai/gpt-4o-mini-transcribe_gpt-4o-2024-11-20
    model_name: openai/gpt-4o-mini-transcribe_gpt-4o-2024-11-20
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"

  - name: openai/gpt-4o-audio-preview-2024-10-01
    model_name: openai/gpt-4o-audio-preview-2024-10-01
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4o-audio-preview-2024-12-17
    model_name: openai/gpt-4o-audio-preview-2024-12-17
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4o-mini-audio-preview-2024-12-17
    model_name: openai/gpt-4o-mini-audio-preview-2024-12-17
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-vision-preview
    model_name: openai/gpt-4-vision-preview
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000  # According to https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
    max_request_length: 128001
    max_sequence_and_generated_tokens_length: 132096
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/gpt-4-1106-vision-preview
    model_name: openai/gpt-4-1106-vision-preview
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000  # According to https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
    max_request_length: 128001
    max_sequence_and_generated_tokens_length: 132096
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  ## GPT-4.5
  - name: openai/gpt-4.5-preview-2025-02-27
    model_name: openai/gpt-4.5-preview-2025-02-27
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  ## o1 Models
  - name: openai/o1-pro-2025-03-19
    model_name: openai/o1-pro-2025-03-19
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"

  - name: openai/o1-pro-2025-03-19-low-reasoning-effort
    model_name: openai/o1-pro-2025-03-19-low-reasoning-effort
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"
      args:
        openai_model_name: o1-pro-2025-03-19
        reasoning_effort: low

  - name: openai/o1-pro-2025-03-19-high-reasoning-effort
    model_name: openai/o1-pro-2025-03-19-high-reasoning-effort
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"
      args:
        openai_model_name: o1-pro-2025-03-19
        reasoning_effort: high

  - name: openai/o1-2024-12-17
    model_name: openai/o1-2024-12-17
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/o1-2024-12-17-low-reasoning-effort
    model_name: openai/o1-2024-12-17-low-reasoning-effort
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o1-2024-12-17
        reasoning_effort: low

  - name: openai/o1-2024-12-17-high-reasoning-effort
    model_name: openai/o1-2024-12-17-high-reasoning-effort
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o1-2024-12-17
        reasoning_effort: high

  - name: openai/o1-preview-2024-09-12
    model_name: openai/o1-preview-2024-09-12
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/o1-mini-2024-09-12
    model_name: openai/o1-mini-2024-09-12
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/o3-mini-2025-01-31
    model_name: openai/o3-mini-2025-01-31
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/o3-mini-2025-01-31-low-reasoning-effort
    model_name: openai/o3-mini-2025-01-31-low-reasoning-effort
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o3-mini-2025-01-31
        reasoning_effort: low

  - name: openai/o3-mini-2025-01-31-high-reasoning-effort
    model_name: openai/o3-mini-2025-01-31-high-reasoning-effort
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 200000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o3-mini-2025-01-31
        reasoning_effort: high

  - name: openai/o3-2025-04-16
    model_name: openai/o3-2025-04-16
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o3
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/o3-2025-04-16-low-reasoning-effort
    model_name: openai/o3-2025-04-16-low-reasoning-effort
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o3
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o3-2025-04-16
        reasoning_effort: low

  - name: openai/o3-2025-04-16-high-reasoning-effort
    model_name: openai/o3-2025-04-16-high-reasoning-effort
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o3
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o3-2025-04-16
        reasoning_effort: high

  - name: openai/o4-mini-2025-04-16
    model_name: openai/o4-mini-2025-04-16
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o4-mini
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: openai/o4-mini-2025-04-16-low-reasoning-effort
    model_name: openai/o4-mini-2025-04-16-low-reasoning-effort
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o4-mini
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o4-mini-2025-04-16
        reasoning_effort: low


  - name: openai/o4-mini-2025-04-16-high-reasoning-effort
    model_name: openai/o4-mini-2025-04-16-high-reasoning-effort
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o4-mini
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"
      args:
        openai_model_name: o4-mini-2025-04-16
        reasoning_effort: high


  - name: openai/o3-pro-2025-06-10-high-reasoning-effort
    model_name: openai/o3-pro-2025-06-10-high-reasoning-effort
    tokenizer_name: openai/cl100k_base
    # Source: https://platform.openai.com/docs/models/o3-pro
    max_sequence_length: 200000
    # TODO: max_output_tokens: 100000
    client_spec:
      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"
      args:
        openai_model_name: o3-pro-2025-06-10
        reasoning_effort: high

  ## GPT-OSS
  - name: together/gpt-oss-20b
    model_name: openai/gpt-oss-20b
    tokenizer_name: openai/o200k_harmony
    # Source: https://platform.openai.com/docs/models/gpt-oss-20b
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/gpt-oss-120b
    model_name: openai/gpt-oss-120b
    tokenizer_name: openai/o200k_harmony
    # Source: https://platform.openai.com/docs/models/gpt-oss-120b
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  ## Text Similarity Models
  # OpenAI similarity embedding models: https://beta.openai.com/docs/guides/embeddings
  # The number of parameters is guessed based on the number of parameters of the
  # corresponding GPT-3 model.

  # As of 2023-11-07, text-embedding-ada-002 is not deprecated:
  # "We recommend using text-embedding-ada-002 for nearly all use cases."
  # Source: https://platform.openai.com/docs/guides/embeddings/what-are-embeddings
  - name: openai/text-embedding-ada-002
    model_name: openai/text-embedding-ada-002
    tokenizer_name: huggingface/gpt2
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  # Text-to-image models
  - name: openai/dall-e-2
    model_name: openai/dall-e-2
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 1000
    client_spec:
      class_name: "helm.clients.image_generation.dalle2_client.DALLE2Client"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"

  - name: openai/dall-e-3
    model_name: openai/dall-e-3
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 1000
    client_spec:
      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"

  - name: openai/dall-e-3-natural
    model_name: openai/dall-e-3-natural
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 1000
    client_spec:
      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"

  - name: openai/dall-e-3-hd
    model_name: openai/dall-e-3-hd
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 1000
    client_spec:
      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"

  - name: openai/dall-e-3-hd-natural
    model_name: openai/dall-e-3-hd-natural
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 1000
    client_spec:
      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"

  # Together
  # The list of models served by Together changes often, to check the latest list, visit:
  # https://docs.together.ai/docs/inference-models
  # You can also check the playground to check that the live models are working:
  # https://api.together.xyz/playground

  ## BigScience
  - name: together/bloom
    deprecated: true  # Removed from Together
    model_name: bigscience/bloom
    tokenizer_name: bigscience/bloom
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/t0pp
    deprecated: true  # Removed from Together
    model_name: bigscience/t0pp
    tokenizer_name: bigscience/T0pp
    max_sequence_length: 1024
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"

  ## Google
  - name: together/t5-11b
    deprecated: true  # Removed from Together
    model_name: google/t5-11b
    tokenizer_name: google/t5-11b
    max_sequence_length: 511
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"

  - name: together/flan-t5-xxl
    deprecated: true  # Removed from Together
    model_name: google/flan-t5-xxl
    tokenizer_name: google/flan-t5-xxl
    max_sequence_length: 511
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"

  - name: together/ul2
    deprecated: true  # Removed from Together
    model_name: google/ul2
    tokenizer_name: google/ul2
    max_sequence_length: 511
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"

  ## Meta
  - name: together/llama-7b
    model_name: meta/llama-7b
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: huggyllama/llama-7b

  - name: together/llama-13b
    model_name: meta/llama-13b
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: huggyllama/llama-13b

  - name: together/llama-30b
    model_name: meta/llama-30b
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: huggyllama/llama-30b

  - name: together/llama-65b
    model_name: meta/llama-65b
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2047 # Subtract 1 tokens to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: huggyllama/llama-65b

  - name: together/llama-2-7b
    model_name: meta/llama-2-7b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/llama-2-7b

  - name: together/llama-2-13b
    model_name: meta/llama-2-13b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/llama-2-13b

  - name: together/llama-2-70b
    model_name: meta/llama-2-70b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/llama-2-70b

  - name: together/llama-3-8b
    model_name: meta/llama-3-8b
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Llama-3-8b-hf

  - name: together/llama-3-8b-instruct-turbo
    model_name: meta/llama-3-8b-instruct-turbo
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Meta-Llama-3-8B-Instruct-Turbo

  - name: together/llama-3-8b-instruct-lite
    model_name: meta/llama-3-8b-instruct-lite
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Meta-Llama-3-8B-Instruct-Lite

  - name: together/llama-3-70b
    model_name: meta/llama-3-70b
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Meta-Llama-3-70B

  - name: together/llama-3-70b-instruct-turbo
    model_name: meta/llama-3-70b-instruct-turbo
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Meta-Llama-3-70B-Instruct-Turbo

  - name: together/llama-3-70b-instruct-lite
    model_name: meta/llama-3-70b-instruct-lite
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Meta-Llama-3-70B-Instruct-Lite

  - name: together/llama-3.1-8b-instruct-turbo
    model_name: meta/llama-3.1-8b-instruct-turbo
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo

  - name: together/llama-3.1-70b-instruct-turbo
    model_name: meta/llama-3.1-70b-instruct-turbo
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo

  - name: together/llama-3.1-405b-instruct-turbo
    model_name: meta/llama-3.1-405b-instruct-turbo
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo

  - name: together/llama-4-scout-17b-16e-instruct
    model_name: meta/llama-4-scout-17b-16e-instruct
    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
    max_sequence_length: 327680
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-4-Scout-17B-16E-Instruct

  - name: together/llama-4-maverick-17b-128e-instruct-fp8
    model_name: meta/llama-4-maverick-17b-128e-instruct-fp8
    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
    max_sequence_length: 524288
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8

  - name: together/llama-3-8b-chat
    model_name: meta/llama-3-8b-chat
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 8182
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-3-8b-chat-hf

  - name: together/llama-3-70b-chat
    model_name: meta/llama-3-70b-chat
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 8182
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-3-70b-chat-hf

  - name: together/llama-3.2-3b-instruct-turbo
    model_name: meta/llama-3.2-3b-instruct-turbo
    tokenizer_name: meta/llama-3.2-3b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-3.2-3B-Instruct-Turbo

  - name: together/llama-3.2-11b-vision-instruct-turbo
    model_name: meta/llama-3.2-11b-vision-instruct-turbo
    tokenizer_name: meta/llama-3.2-11b-vision-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo

  - name: together/llama-3.2-90b-vision-instruct-turbo
    model_name: meta/llama-3.2-90b-vision-instruct-turbo
    tokenizer_name: meta/llama-3.2-11b-vision-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo

  - name: together/llama-3.3-70b-instruct-turbo
    model_name: meta/llama-3.3-70b-instruct-turbo
    tokenizer_name: meta/llama-3.3-70b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: meta-llama/Llama-3.3-70B-Instruct-Turbo

  - name: together/llama-guard-7b
    model_name: meta/llama-guard-7b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/llama-guard-7b

  - name: together/llama-guard-2-8b
    model_name: meta/llama-guard-2-8b
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 4094
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/llamaguard-2-8b

  - name: together/llama-guard-3-8b
    model_name: meta/llama-guard-3-8b
    tokenizer_name: meta/llama-3.1-8b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: meta-llama/Meta-Llama-Guard-3-8B

  # NVIDIA  
  - name: together/llama-3.1-nemotron-70b-instruct
    model_name: nvidia/llama-3.1-nemotron-70b-instruct
    tokenizer_name: nvidia/llama-3.1-nemotron-70b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: nvidia/Llama-3.1-Nemotron-70B-Instruct-HF

  # 01.AI
  - name: together/yi-6b
    model_name: 01-ai/yi-6b
    tokenizer_name: 01-ai/Yi-6B
    max_sequence_length: 4095
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: zero-one-ai/Yi-6B

  - name: together/yi-34b
    model_name: 01-ai/yi-34b
    tokenizer_name: 01-ai/Yi-6B
    max_sequence_length: 4095
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: zero-one-ai/Yi-34B

  - name: together/yi-6b-chat
    model_name: 01-ai/yi-6b-chat
    tokenizer_name: 01-ai/Yi-6B
    max_sequence_length: 4095
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: zero-one-ai/Yi-6B-Chat

  - name: together/yi-34b-chat
    model_name: 01-ai/yi-34b-chat
    tokenizer_name: 01-ai/Yi-6B
    max_sequence_length: 4095
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: zero-one-ai/Yi-34B-Chat

  - name: 01-ai/yi-large
    model_name: 01-ai/yi-large
    tokenizer_name: 01-ai/Yi-6B  # Actual tokenizer is publicly unavailable, so use a substitute
    max_sequence_length: 16000
    client_spec:
      class_name: "helm.clients.yi_client.YiChatClient"

  - name: 01-ai/yi-large-preview
    model_name: 01-ai/yi-large-preview
    tokenizer_name: 01-ai/Yi-6B  # Actual tokenizer is publicly unavailable, so use a substitute
    max_sequence_length: 16000
    client_spec:
      class_name: "helm.clients.yi_client.YiChatClient"


  # Allen Institute for AI
  - name: together/olmo-7b
    model_name: allenai/olmo-7b
    tokenizer_name: allenai/olmo-7b
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/olmo-7b-twin-2t
    model_name: allenai/olmo-7b-twin-2t
    tokenizer_name: allenai/olmo-7b
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/olmo-7b-instruct
    model_name: allenai/olmo-7b-instruct
    tokenizer_name: allenai/olmo-7b
    max_sequence_length: 2047
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: huggingface/olmo-1.7-7b
    model_name: allenai/olmo-1.7-7b
    tokenizer_name: allenai/OLMo-1.7-7B-hf
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: allenai/OLMo-1.7-7B-hf

  - name: huggingface/olmo-2-1124-7b-instruct
    model_name: allenai/olmo-2-1124-7b-instruct
    tokenizer_name: allenai/olmo-2-1124-7b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/olmo-2-1124-13b-instruct
    model_name: allenai/olmo-2-1124-13b-instruct
    tokenizer_name: allenai/olmo-2-1124-7b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/olmo-2-0325-32b-instruct
    model_name: allenai/olmo-2-0325-32b-instruct
    tokenizer_name: allenai/olmo-2-0325-32b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/olmoe-1b-7b-0125-instruct
    model_name: allenai/olmoe-1b-7b-0125-instruct
    tokenizer_name: allenai/olmoe-1b-7b-0125-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  ## MistralAI
  - name: together/mistral-7b-v0.1
    model_name: mistralai/mistral-7b-v0.1
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: mistralai/Mistral-7B-v0.1

  - name: together/mistral-7b-instruct-v0.1
    model_name: mistralai/mistral-7b-instruct-v0.1
    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.1
    max_sequence_length: 4000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/mistral-7b-instruct-v0.2
    model_name: mistralai/mistral-7b-instruct-v0.2
    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.2
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: huggingface/mistral-7b-instruct-v0.3
    model_name: mistralai/mistral-7b-instruct-v0.3-hf
    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.3
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: mistralai/Mistral-7B-Instruct-v0.3

  - name: together/mistral-7b-instruct-v0.3
    model_name: mistralai/mistral-7b-instruct-v0.3
    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.3
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      

  - name: together/mixtral-8x7b-32kseqlen
    model_name: mistralai/mixtral-8x7b-32kseqlen
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: mistralai/mixtral-8x7b-32kseqlen

  - name: together/mixtral-8x7b-instruct-v0.1
    model_name: mistralai/mixtral-8x7b-instruct-v0.1
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/mixtral-8x22b
    model_name: mistralai/mixtral-8x22b
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 65535
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"

  - name: together/mixtral-8x22b-instruct-v0.1
    model_name: mistralai/mixtral-8x22b-instruct-v0.1
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 65535
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"


  ## Snowflake
  - name: together/snowflake-arctic-instruct
    model_name: snowflake/snowflake-arctic-instruct
    tokenizer_name: snowflake/snowflake-arctic-instruct
    max_sequence_length: 4000  # Lower than 4096 because of chat tokens
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  ## Stanford
  - name: together/alpaca-7b
    model_name: stanford/alpaca-7b
    tokenizer_name: hf-internal-testing/llama-tokenizer
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/alpaca-7b

  ## Tiiuae
  - name: together/falcon-7b
    model_name: tiiuae/falcon-7b
    tokenizer_name: tiiuae/falcon-7b
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/falcon-7b

  - name: together/falcon-7b-instruct
    model_name: tiiuae/falcon-7b-instruct
    tokenizer_name: tiiuae/falcon-7b
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/falcon-7b-instruct

  - name: together/falcon-40b
    model_name: tiiuae/falcon-40b
    tokenizer_name: tiiuae/falcon-7b
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/falcon-40b

  - name: together/falcon-40b-instruct
    model_name: tiiuae/falcon-40b-instruct
    tokenizer_name: tiiuae/falcon-7b
    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/falcon-40b-instruct

  ## Together
  # These are models fine-tuned by Together (and not simply hosted by Together).
  - name: together/gpt-jt-6b-v1
    model_name: together/gpt-jt-6b-v1
    tokenizer_name: EleutherAI/gpt-j-6B
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/GPT-JT-6B-v1

  - name: together/gpt-neoxt-chat-base-20b
    model_name: together/gpt-neoxt-chat-base-20b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/GPT-NeoXT-Chat-Base-20B

  - name: together/redpajama-incite-base-3b-v1
    model_name: together/redpajama-incite-base-3b-v1
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/RedPajama-INCITE-Base-3B-v1

  - name: together/redpajama-incite-instruct-3b-v1
    model_name: together/redpajama-incite-instruct-3b-v1
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/RedPajama-INCITE-Instruct-3B-v1

  - name: together/redpajama-incite-base-7b
    model_name: together/redpajama-incite-base-7b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/RedPajama-INCITE-7B-Base

  - name: together/redpajama-incite-instruct-7b
    model_name: together/redpajama-incite-instruct-7b
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/RedPajama-INCITE-7B-Instruct

  ## Z.ai
  - name: together/glm-4.5-air-fp8
    model_name: zai-org/glm-4.5-air-fp8
    tokenizer_name: zai-org/glm-4.5-air-fp8
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        parse_thinking: true

  - name: thudm/cogview2
    model_name: thudm/cogview2
    tokenizer_name: openai/clip-vit-large-patch14
    max_sequence_length: 75
    client_spec:
      class_name: "helm.clients.image_generation.cogview2_client.CogView2Client"
    window_service_spec:
      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"

  ## Yandex
  - name: together/yalm
    deprecated: true  # Removed from Together
    model_name: yandex/yalm
    tokenizer_name: Yandex/yalm
    max_sequence_length: 2048
    max_request_length: 2049
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.yalm_window_service.YaLMWindowService"

  # Writer
  - name: writer/palmyra-base
    model_name: writer/palmyra-base
    tokenizer_name: writer/gpt2
    max_sequence_length: 2048
    max_sequence_and_generated_tokens_length: 2048
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/palmyra-large
    model_name: writer/palmyra-large
    tokenizer_name: writer/gpt2
    max_sequence_length: 2048
    max_sequence_and_generated_tokens_length: 2048
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/silk-road
    model_name: writer/silk-road
    tokenizer_name: writer/gpt2
    max_sequence_length: 8192
    max_sequence_and_generated_tokens_length: 8192
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/palmyra-x
    model_name: writer/palmyra-x
    tokenizer_name: writer/gpt2
    max_sequence_length: 8192
    max_sequence_and_generated_tokens_length: 8192
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/palmyra-x-v2
    model_name: writer/palmyra-x-v2
    tokenizer_name: writer/gpt2
    max_sequence_length: 6000
    max_sequence_and_generated_tokens_length: 7024
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/palmyra-x-v3
    model_name: writer/palmyra-x-v3
    tokenizer_name: writer/gpt2
    max_sequence_length: 6000
    max_sequence_and_generated_tokens_length: 7024
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/palmyra-x-32k
    model_name: writer/palmyra-x-32k
    tokenizer_name: writer/gpt2
    max_sequence_length: 28000
    max_sequence_and_generated_tokens_length: 30048
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraClient"

  - name: writer/palmyra-vision-003
    model_name: writer/palmyra-vision-003
    tokenizer_name: writer/gpt2
    max_sequence_length: 2048
    max_sequence_and_generated_tokens_length: 2048
    client_spec:
      class_name: "helm.clients.vision_language.palmyra_vision_client.PalmyraVisionClient"

  - name: writer/palmyra-x-004
    model_name: writer/palmyra-x-004
    # Actual tokenizer is Llama 2, but it cannot be used in HELM due to this issue:
    # https://github.com/stanford-crfm/helm/issues/2467
    # Work around by using Llama 3 tokenizer for now.
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraChatClient"

  - name: writer/palmyra-x5
    model_name: writer/palmyra-x5
    # See tokenizer comment for writer/palmyra-x-004
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 1000000
    client_spec:
      class_name: "helm.clients.writer_client.WriterClient"

  - name: amazon/palmyra-x5-v1-bedrock
    model_name: writer/palmyra-x5-v1-bedrock
    # See tokenizer comment for writer/palmyra-x-004
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 1040000
    client_spec:
      class_name: "helm.clients.bedrock_client.BedrockPalmyraClient"
      args:
        bedrock_model_id: us.writer.palmyra-x5-v1:0
        region: us-west-2

  - name: writer/palmyra-med-32k
    model_name: writer/palmyra-med-32k
    # Palmyra-Med uses the "<|end_of_text|>" as the end of text token, which is used by meta/llama-3-8b,
    # rather than "<|eot_id|>", which is used by meta/llama-3-8b-instruct
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraChatClient"

  - name: writer/palmyra-med
    model_name: writer/palmyra-med
    tokenizer_name: meta/llama-3-8b
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.writer_client.WriterClient"

  - name: writer/palmyra-fin-32k
    model_name: writer/palmyra-fin-32k
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraChatClient"

  - name: writer/palmyra-fin
    model_name: writer/palmyra-fin
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.palmyra_client.PalmyraChatClient"


  # xAI

  - name: xai/grok-3-beta
    model_name: xai/grok-3-beta
    tokenizer_name: xai/grok-3-beta
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: xai/grok-3-mini-beta
    model_name: xai/grok-3-mini-beta
    tokenizer_name: xai/grok-3-mini-beta
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: xai/grok-4-0709
    model_name: xai/grok-4-0709
    tokenizer_name: xai/grok-4-0709
    max_sequence_length: 256000
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: xai/grok-4-fast-reasoning
    model_name: xai/grok-4-fast-reasoning
    tokenizer_name: xai/grok-4-fast-reasoning
    max_sequence_length: 2000000
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: xai/grok-4-fast-non-reasoning
    model_name: xai/grok-4-fast-non-reasoning
    tokenizer_name: xai/grok-4-fast-non-reasoning
    max_sequence_length: 2000000
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: xai/grok-4-1-fast-reasoning
    model_name: xai/grok-4-1-fast-reasoning
    tokenizer_name: xai/grok-4-1-fast-reasoning
    max_sequence_length: 2000000
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  - name: xai/grok-4-1-fast-non-reasoning
    model_name: xai/grok-4-1-fast-non-reasoning
    tokenizer_name: xai/grok-4-1-fast-non-reasoning
    max_sequence_length: 2000000
    client_spec:
      class_name: "helm.clients.grok_client.GrokChatClient"
    window_service_spec:
      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"

  # Qwen

  - name: together/qwen-7b
    model_name: qwen/qwen-7b
    tokenizer_name: qwen/qwen-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: togethercomputer/Qwen-7B

  - name: together/qwen1.5-7b
    model_name: qwen/qwen1.5-7b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: Qwen/Qwen1.5-7B

  - name: together/qwen1.5-14b
    model_name: qwen/qwen1.5-14b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: Qwen/Qwen1.5-14B

  - name: together/qwen1.5-32b
    model_name: qwen/qwen1.5-32b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: Qwen/Qwen1.5-32B

  - name: together/qwen1.5-72b
    model_name: qwen/qwen1.5-72b
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherClient"
      args:
        together_model: Qwen/Qwen1.5-72B

  - name: together/qwen1.5-7b-chat
    model_name: qwen/qwen1.5-7b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen1.5-14b-chat
    model_name: qwen/qwen1.5-14b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen1.5-32b-chat
    model_name: qwen/qwen1.5-32b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen1.5-72b-chat
    model_name: qwen/qwen1.5-72b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen1.5-110b-chat
    model_name: qwen/qwen1.5-110b-chat
    tokenizer_name: qwen/qwen1.5-7b
    max_sequence_length: 32767
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen2-72b-instruct
    model_name: qwen/qwen2-72b-instruct
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen2.5-7b-instruct-turbo
    model_name: qwen/qwen2.5-7b-instruct-turbo
    tokenizer_name: qwen/qwen2.5-7b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen2.5-72b-instruct-turbo
    model_name: qwen/qwen2.5-72b-instruct-turbo
    tokenizer_name: qwen/qwen2.5-7b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen3-235b-a22b-fp8-tput
    model_name: qwen/qwen3-235b-a22b-fp8-tput
    tokenizer_name: qwen/qwen3-235b-a22b
    max_sequence_length: 40960
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        parse_thinking: true

  - name: together/qwen3-next-80b-a3b-instruct
    model_name: qwen/qwen3-next-80b-a3b-instruct
    tokenizer_name: qwen/qwen3-next-80b-a3b-instruct
    max_sequence_length: 262144
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: together/qwen3-next-80b-a3b-thinking
    model_name: qwen/qwen3-next-80b-a3b-thinking
    tokenizer_name: qwen/qwen3-next-80b-a3b-thinking
    max_sequence_length: 262144
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        parse_thinking: true

  - name: together/qwen3-235b-a22b-instruct-2507-fp8
    model_name: qwen/qwen3-235b-a22b-instruct-2507-fp8
    tokenizer_name: qwen/qwen3-235b-a22b-instruct-2507-fp8
    max_sequence_length: 262144
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        together_model: Qwen/Qwen3-235B-A22B-Instruct-2507-tput

  - name: together/qwen3-235b-a22b-thinking-2507
    model_name: qwen/qwen3-235b-a22b-thinking-2507
    tokenizer_name: qwen/qwen3-235b-a22b-thinking-2507
    max_sequence_length: 262144
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"
      args:
        # The model name is case sensitive on Together AI for this model
        together_model: Qwen/Qwen3-235B-A22B-Thinking-2507

  - name: huggingface/qwen2.5-7b-instruct-4bit
    model_name: qwen/qwen2.5-7b-instruct
    tokenizer_name: qwen/qwen2.5-7b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: Qwen/Qwen2.5-7B-Instruct
        torch_dtype: "float16"
        quantization_config:
          load_in_4bit: true
        attn_implementation: "flash_attention_2"

  - name: huggingface/qwen2.5-7b-instruct
    model_name: qwen/qwen2.5-7b-instruct
    tokenizer_name: qwen/qwen2.5-7b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: Qwen/Qwen2.5-7B-Instruct

  - name: huggingface/smollm2-135m
    model_name: huggingface/smollm2-135m
    tokenizer_name: huggingface/smollm2-135m
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
      args:
        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-135M

  - name: huggingface/smollm2-360m
    model_name: huggingface/smollm2-360m
    tokenizer_name: huggingface/smollm2-135m
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
      args:
        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-360M

  - name: huggingface/smollm2-1.7b
    model_name: huggingface/smollm2-1.7b
    tokenizer_name: huggingface/smollm2-135m
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
      args:
        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-1.7B

  - name: huggingface/smollm2-135m-instruct
    model_name: huggingface/smollm2-135m-instruct
    tokenizer_name: huggingface/smollm2-135m-instruct
    max_sequence_length: 328192768
    client_spec:
      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
      args:
        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-135M-Instruct

  - name: huggingface/smollm2-360m-instruct
    model_name: huggingface/smollm2-360m-instruct
    tokenizer_name: huggingface/smollm2-135m-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
      args:
        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-360M-Instruct

  - name: huggingface/smollm2-1.7b-instruct
    model_name: huggingface/smollm2-1.7b-instruct
    tokenizer_name: huggingface/smollm2-135m-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
      args:
        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-1.7B-Instruct

  - name: together/qwq-32b-preview
    model_name: qwen/qwq-32b-preview
    tokenizer_name: qwen/qwq-32b-preview
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.together_client.TogetherChatClient"

  - name: huggingface/qwen-vl
    model_name: qwen/qwen-vl
    tokenizer_name: qwen/qwen-vl
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"

  - name: huggingface/qwen-vl-chat
    model_name: qwen/qwen-vl-chat
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"

  - name: huggingface/qwen2-vl-7b-instruct
    model_name: qwen/qwen2-vl-7b-instruct
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"

  - name: huggingface/qwen2-vl-72b-instruct
    model_name: qwen/qwen2-vl-72b-instruct
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"

  - name: huggingface/qwen2.5-vl-3b-instruct
    model_name: qwen/qwen2.5-vl-3b-instruct
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"

  - name: huggingface/qwen2.5-vl-7b-instruct
    model_name: qwen/qwen2.5-vl-7b-instruct
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"

  - name: huggingface/qwen2.5-vl-32b-instruct
    model_name: qwen/qwen2.5-vl-32b-instruct
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"

  - name: huggingface/qwen2.5-vl-72b-instruct
    model_name: qwen/qwen2.5-vl-72b-instruct
    tokenizer_name: qwen/qwen-vl-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"

  - name: huggingface/qwen-audio-chat
    model_name: qwen/qwen-audio-chat
    tokenizer_name: qwen/qwen-audio-chat
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.audio_language.qwen_audiolm_client.QwenAudioLMClient"

  - name: huggingface/qwen2-audio-7b-instruct
    model_name: qwen/qwen2-audio-7b-instruct
    tokenizer_name: qwen/qwen2-audio-instruct
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.audio_language.qwen2_audiolm_client.Qwen2AudioLMClient"

  - name: huggingface/qwen2.5-omni-7b
    model_name: qwen/qwen2.5-omni-7b
    tokenizer_name: qwen/qwen2.5-omni-7b
    max_sequence_length: 8191
    client_spec:
      class_name: "helm.clients.audio_language.qwen2_5_omni_client.Qwen2_5OmniAudioLMClient"

# Reka
  - name: reka/reka-core
    model_name: reka/reka-core
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"
  
  - name: reka/reka-core-20240415
    model_name: reka/reka-core-20240415
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"
  
  - name: reka/reka-core-20240501
    model_name: reka/reka-core-20240501
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"

  - name: reka/reka-flash
    model_name: reka/reka-flash
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"

  - name: reka/reka-flash-20240226
    model_name: reka/reka-flash-20240226
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"

  - name: reka/reka-edge
    model_name: reka/reka-edge
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 64000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"

  - name: reka/reka-edge-20240208
    model_name: reka/reka-edge-20240208
    tokenizer_name: openai/cl100k_base
    max_sequence_length: 64000
    client_spec:
      class_name: "helm.clients.reka_client.RekaClient"

  # Upstage
  - name: upstage/solar-pro-241126
    model_name: upstage/solar-pro-241126
    tokenizer_name: upstage/solar-pro-preview-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.upstage_client.UpstageChatClient"

# Diva Llama
  - name: huggingface/diva-llama
    model_name: stanford/diva-llama
    # TODO: Set the right tokenizer
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.audio_language.diva_llama_client.DivaLlamaClient"

# LLaMA-Omni
  - name: ictnlp/llama-3.1-8b-omni
    model_name: ictnlp/llama-3.1-8b-omni
    tokenizer_name: ictnlp/llama-3.1-8b-omni
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.audio_language.llama_omni_client.LlamaOmniAudioLMClient"

# IBM Granite
  - name: huggingface/granite-3.0-2b-base
    model_name: ibm-granite/granite-3.0-2b-base
    tokenizer_name: ibm-granite/granite-3.0-2b-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-2b-base 
    
  - name: huggingface/granite-3.0-2b-instruct
    model_name: ibm-granite/granite-3.0-2b-instruct
    tokenizer_name: ibm-granite/granite-3.0-2b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-2b-instruct

  - name: huggingface/granite-3.0-8b-instruct
    model_name: ibm-granite/granite-3.0-8b-instruct
    tokenizer_name: ibm-granite/granite-3.0-8b-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-8b-instruct
  
  - name: huggingface/granite-3.0-8b-base
    model_name: ibm-granite/granite-3.0-8b-base
    tokenizer_name: ibm-granite/granite-3.0-8b-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-8b-base

  - name: huggingface/granite-3.0-3b-a800m-instruct
    model_name: ibm-granite/granite-3.0-3b-a800m-instruct
    tokenizer_name: ibm-granite/granite-3.0-3b-a800m-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-3b-a800m-instruct

  - name: huggingface/granite-3.0-3b-a800m-base
    model_name: ibm-granite/granite-3.0-3b-a800m-base
    tokenizer_name: ibm-granite/granite-3.0-3b-a800m-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-3b-a800m-base

  - name: huggingface/granite-3.0-1b-a400m-instruct
    model_name: ibm-granite/granite-3.0-1b-a400m-instruct
    tokenizer_name: ibm-granite/granite-3.0-1b-a400m-instruct
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-1b-a400m-instruct

  - name: huggingface/granite-3.0-1b-a400m-base
    model_name: ibm-granite/granite-3.0-1b-a400m-base
    tokenizer_name: ibm-granite/granite-3.0-1b-a400m-base
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-3.0-1b-a400m-base

  - name: huggingface/granite-4.0-micro
    model_name: ibm/granite-4.0-micro
    tokenizer_name: ibm/granite-4.0-micro
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-4.0-micro

  - name: huggingface/granite-4.0-h-small
    model_name: ibm/granite-4.0-h-small
    tokenizer_name: ibm/granite-4.0-h-small
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ibm-granite/granite-4.0-h-small

# Maritaca AI
  - name: huggingface/sabia-7b
    model_name: maritaca-ai/sabia-7b
    tokenizer_name: maritaca-ai/sabia-7b
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
    args:
      pretrained_model_name_or_path: maritaca-ai/sabia-7b

  - name: maritaca-ai/sabiazinho-3
    model_name: maritaca-ai/sabiazinho-3
    tokenizer_name: maritaca-ai/sabia-2-tokenizer-medium
    max_sequence_length: 32000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: maritaca-ai/sabia-3
    model_name: maritaca-ai/sabia-3
    tokenizer_name: maritaca-ai/sabia-2-tokenizer-medium
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

  - name: maritaca-ai/sabia-3.1-2025-05-08
    model_name: maritaca-ai/sabia-3.1-2025-05-08
    tokenizer_name: maritaca-ai/sabia-2-tokenizer-medium
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openai_client.OpenAIClient"

# Granite-3.1-8b-base
  - name: huggingface/granite-3.1-8b-base
    model_name: ibm-granite/granite-3.1-8b-base
    tokenizer_name: ibm-granite/granite-3.1-8b-base
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-8b-base

# Granite-3.1-8b-instruct
  - name: huggingface/granite-3.1-8b-instruct
    model_name: ibm-granite/granite-3.1-8b-instruct
    tokenizer_name: ibm-granite/granite-3.1-8b-instruct
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-8b-instruct

# Granite-3.1-2b-instruct
  - name: huggingface/granite-3.1-2b-instruct
    model_name: ibm-granite/granite-3.1-2b-instruct
    tokenizer_name: ibm-granite/granite-3.1-2b-instruct
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-2b-instruct

# Granite-3.1-2b-base
  - name: huggingface/granite-3.1-2b-base
    model_name: ibm-granite/granite-3.1-2b-base
    tokenizer_name: ibm-granite/granite-3.1-2b-base
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-2b-base

# Granite-3.1-3b-a800m-instruct
  - name: huggingface/granite-3.1-3b-a800m-instruct
    model_name: ibm-granite/granite-3.1-3b-a800m-instruct
    tokenizer_name: ibm-granite/granite-3.1-3b-a800m-instruct
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-3b-a800m-instruct

# Granite-3.1-3b-a800m-base
  - name: huggingface/granite-3.1-3b-a800m-base
    model_name: ibm-granite/granite-3.1-3b-a800m-base
    tokenizer_name: ibm-granite/granite-3.1-3b-a800m-base
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-3b-a800m-base

# Granite-3.1-1b-a400m-instruct
  - name: huggingface/granite-3.1-1b-a400m-instruct
    model_name: ibm-granite/granite-3.1-1b-a400m-instruct
    tokenizer_name: ibm-granite/granite-3.1-1b-a400m-instruct
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-1b-a400m-instruct

# Granite-3.1-1b-a400m-base
  - name: huggingface/granite-3.1-1b-a400m-base
    model_name: ibm-granite/granite-3.1-1b-a400m-base
    tokenizer_name: ibm-granite/granite-3.1-1b-a400m-base
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: ibm-granite/granite-3.1-1b-a400m-base

# DeepSeek-R1-Distill-Llama-3.1-8b
  - name: huggingface/DeepSeek-R1-Distill-Llama-8B
    model_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B
    tokenizer_name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: deepseek-ai/DeepSeek-R1-Distill-Llama-8B

# deepseek-ai/deepseek-coder-6.7b-instruct
  - name: huggingface/deepseek-coder-6.7b-instruct
    model_name: deepseek-ai/deepseek-coder-6.7b-instruct
    tokenizer_name: deepseek-ai/deepseek-coder-6.7b-instruct
    max_sequence_length: 128000
    client_spec:
        class_name: "helm.clients.huggingface_client.HuggingFaceClient"
        args:
            pretrained_model_name_or_path: deepseek-ai/deepseek-coder-6.7b-instruct

  # AceGPT-v2
  - name: huggingface/acegpt-v2-8b-chat
    model_name: freedomintelligence/acegpt-v2-8b-chat
    tokenizer_name: freedomintelligence/acegpt-v2-8b-chat
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/acegpt-v2-32b-chat
    model_name: freedomintelligence/acegpt-v2-32b-chat
    tokenizer_name: freedomintelligence/acegpt-v2-32b-chat
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/acegpt-v2-70b-chat
    model_name: freedomintelligence/acegpt-v2-70b-chat
    tokenizer_name: freedomintelligence/acegpt-v2-70b-chat
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  # ALLaM
  - name: huggingface/allam-7b-instruct-preview
    model_name: allam-ai/allam-7b-instruct-preview
    tokenizer_name: allam-ai/allam-7b-instruct-preview
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  # SILMA
  - name: huggingface/silma-9b-instruct-v1.0
    model_name: silma-ai/silma-9b-instruct-v1.0
    tokenizer_name: silma-ai/silma-9b-instruct-v1.0
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  # Jais Family
  #
  # NOTE: Jais Family models require `transformers<=4.52.3`.
  # On more recent versions of transformers, one of the following errors might occur:
  #
  #   File "/path/to//site-packages/transformers/models/gemma3n/configuration_gemma3n.py", line 31, in <module>
  #     from timm.data import ImageNetInfo, infer_imagenet_subset
  # ImportError: cannot import name 'ImageNetInfo' from 'timm.data' (/path/to/site-packages/timm/data/__init__.py)
  #
  #   File "/path/to/.cache/huggingface/modules/transformers_modules/inceptionai/jais-family-590m-chat/90ac4769212b4964c6e81e183140224628228365/modeling_jais.py", line 899, in forward
  #     past_length = past_key_values[0][0].size(-2)
  # AttributeError: 'NoneType' object has no attribute 'size'

  - name: huggingface/jais-family-590m-chat
    model_name: inceptionai/jais-family-590m-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 2000  # Reduced from 2048 to account for chat template tokens
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        trust_remote_code: true
        revision: 90ac4769212b4964c6e81e183140224628228365

  - name: huggingface/jais-family-1p3b-chat
    model_name: inceptionai/jais-family-1p3b-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 2000  # Reduced from 2048 to account for chat template tokens
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        trust_remote_code: true
        revision: 4b93176e2cb00f369b3bc0a8786e4cf16260c804

  - name: huggingface/jais-family-2p7b-chat
    model_name: inceptionai/jais-family-2p7b-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 2000  # Reduced from 2048 to account for chat template tokens
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        trust_remote_code: true
        revision: b2bf5d1bcd969ce868f66fb1ad8c3480289ea6b2

  - name: huggingface/jais-family-6p7b-chat
    model_name: inceptionai/jais-family-6p7b-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 2000  # Reduced from 2048 to account for chat template tokens
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        trust_remote_code: true
        revision: 683805efe6126c6536feb4aa23317e70222ac94c

  - name: huggingface/jais-family-13b-chat
    model_name: inceptionai/jais-family-13b-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 2000  # Reduced from 2048 to account for chat template tokens
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        trust_remote_code: true
        revision: 0ef8b4f80429609890816d912b331d3b95864707

  - name: huggingface/jais-family-30b-8k-chat
    model_name: inceptionai/jais-family-30b-8k-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        trust_remote_code: true
        revision: dab185164dd3b79ec9201d7f4cf878ce91ae7e14

  - name: huggingface/jais-family-30b-16k-chat
    model_name: inceptionai/jais-family-30b-16k-chat
    tokenizer_name: inceptionai/jais-family-590m-chat
    max_sequence_length: 16384
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto
        trust_remote_code: true
        revision: 369f88eeee4d313155f1b1dca4ebec90f9f9f2a4

  # Jais Adapter
  - name: huggingface/jais-adapted-7b-chat
    model_name: inceptionai/jais-adapted-7b-chat
    tokenizer_name: inceptionai/jais-adapted-7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/jais-adapted-13b-chat
    model_name: inceptionai/jais-adapted-13b-chat
    tokenizer_name: inceptionai/jais-adapted-13b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/jais-adapted-70b-chat
    model_name: inceptionai/jais-adapted-70b-chat
    tokenizer_name: inceptionai/jais-adapted-7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/falcon3-1b-instruct
    model_name: tiiuae/falcon3-1b-instruct
    tokenizer_name: tiiuae/falcon3-1b-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/falcon3-3b-instruct
    model_name: tiiuae/falcon3-3b-instruct
    tokenizer_name: tiiuae/falcon3-1b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"

  - name: huggingface/falcon3-7b-instruct
    model_name: tiiuae/falcon3-7b-instruct
    tokenizer_name: tiiuae/falcon3-1b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

  - name: huggingface/falcon3-10b-instruct
    model_name: tiiuae/falcon3-10b-instruct
    tokenizer_name: tiiuae/falcon3-1b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        device_map: auto

# IBM WatsonX
  - name: ibm/llama-3.3-70b-instruct
    model_name: meta/llama-3.3-70b-instruct
    tokenizer_name: meta/llama-3.3-70b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmChatClient"
      args:
        watsonx_model_name: meta-llama/llama-3-3-70b-instruct
        region: Dallas

  - name: ibm/granite-3-2b-instruct
    model_name: ibm/granite-3.1-2b-instruct
    tokenizer_name: ibm-granite/granite-3.1-2b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmChatClient"
      args:
        watsonx_model_name: ibm/granite-3-2b-instruct
        region: Dallas

  - name: ibm/granite-3-8b-instruct
    model_name: ibm/granite-3.1-8b-instruct
    tokenizer_name: ibm-granite/granite-3.1-8b-instruct
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmChatClient"
      args:
        watsonx_model_name: ibm/granite-3-8b-instruct
        region: Dallas

  - name: ibm/granite-13b-instruct-v2
    model_name: ibm/granite-13b-instruct-v2
    tokenizer_name: EleutherAI/gpt-neox-20b
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmTextClient"
      args:
        watsonx_model_name: ibm/granite-13b-instruct-v2
        region: Dallas

  - name: ibm/granite-20b-code-instruct-8k
    model_name: ibm/granite-20b-code-instruct-8k
    tokenizer_name: ibm-granite/granite-20b-code-instruct-8k
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmChatClient"
      args:
        watsonx_model_name: ibm/granite-20b-code-instruct
        region: Dallas

  - name: ibm/granite-34b-code-instruct
    model_name: ibm/granite-34b-code-instruct
    tokenizer_name: ibm-granite/granite-34b-code-instruct-8k
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmChatClient"
      args:
        watsonx_model_name: ibm/granite-34b-code-instruct
        region: Dallas

  - name: ibm/granite-3b-code-instruct
    model_name: ibm/granite-3b-code-instruct
    tokenizer_name: ibm-granite/granite-3b-code-instruct-128k
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmTextClient"
      args:
        watsonx_model_name: ibm/granite-3b-code-instruct
        region: Dallas

  - name: ibm/granite-8b-code-instruct
    model_name: ibm/granite-8b-code-instruct
    tokenizer_name: ibm-granite/granite-8b-code-instruct-128k
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.ibm_client.IbmTextClient"
      args:
        watsonx_model_name: ibm/granite-8b-code-instruct
        region: Dallas

  - name: ibm/granite-3.3-8b-instruct
    model_name: ibm/granite-3.3-8b-instruct
    tokenizer_name: ibm/granite-3.3-8b-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.ibm_client.IbmTextClient"
      args:
        watsonx_model_name: ibm/granite-3-3-8b-instruct
        region: Dallas

  # Vietnamese
  - name: ura-hcmut/ura-llama-2.1-8b
    model_name: ura-hcmut/ura-llama-2.1-8b
    tokenizer_name: meta/llama-3.1-8b-instruct
    max_sequence_length: 131072
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/ura-llama-2.1-8b

  - name: ura-hcmut/ura-llama-2-8b
    model_name: ura-hcmut/ura-llama-2-8b
    tokenizer_name: meta/llama-3-8b-instruct
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/ura-llama-2-8b

  - name: ura-hcmut/ura-llama-7b
    model_name: ura-hcmut/ura-llama-7b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/ura-llama-7b

  - name: ura-hcmut/ura-llama-13b
    model_name: ura-hcmut/ura-llama-13b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/ura-llama-13b

  - name: ura-hcmut/ura-llama-70b
    model_name: ura-hcmut/ura-llama-70b
    tokenizer_name: meta-llama/Llama-2-7b-hf
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/ura-llama-70b

  - name: ura-hcmut/GemSUra-7B
    model_name: ura-hcmut/GemSUra-7B
    tokenizer_name: google/gemma-2b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/GemSUra-7B

  - name: ura-hcmut/GemSUra-2B
    model_name: ura-hcmut/GemSUra-2B
    tokenizer_name: google/gemma-2b
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/GemSUra-2B

  - name: ura-hcmut/MixSUra
    model_name: ura-hcmut/MixSUra
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: ura-hcmut/MixSUra

  - name: vilm/vinallama-7b-chat
    model_name: vilm/vinallama-7b-chat
    tokenizer_name: vilm/vinallama-7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/vinallama-7b-chat

  - name: vilm/vinallama-2.7b-chat
    model_name: vilm/vinallama-2.7b-chat
    tokenizer_name: vilm/vinallama-2.7b-chat
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/vinallama-2.7b-chat

  - name: vilm/vietcuna-7b-v3
    model_name: vilm/vietcuna-7b-v3
    tokenizer_name: vilm/vietcuna-7b-v3
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/vietcuna-7b-v3

  - name: vilm/vietcuna-3b-v2
    model_name: vilm/vietcuna-3b-v2
    tokenizer_name: vilm/vietcuna-7b-v3
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/vietcuna-3b-v2

  - name: vilm/Quyen-v0.1
    model_name: vilm/Quyen-v0.1
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/Quyen-v0.1

  - name: vilm/Quyen-Plus-v0.1
    model_name: vilm/Quyen-Plus-v0.1
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/Quyen-Plus-v0.1

  - name: vilm/Quyen-Pro-v0.1
    model_name: vilm/Quyen-Pro-v0.1
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/Quyen-Pro-v0.1
  
  - name: vilm/Quyen-Pro-Max-v0.1
    model_name: vilm/Quyen-Pro-Max-v0.1
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/Quyen-Pro-Max-v0.1

  - name: vilm/Quyen-Mini-v0.1
    model_name: vilm/Quyen-Mini-v0.1
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/Quyen-Mini-v0.1

  - name: vilm/Quyen-SE-v0.1
    model_name: vilm/Quyen-SE-v0.1
    tokenizer_name: qwen/qwen2-72b-instruct
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vilm/Quyen-SE-v0.1

  - name: Viet-Mistral/Vistral-7B-Chat
    model_name: Viet-Mistral/Vistral-7B-Chat
    tokenizer_name: Viet-Mistral/Vistral-7B-Chat
    max_sequence_length: 32768
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: Viet-Mistral/Vistral-7B-Chat

  - name: vinai/PhoGPT-7B5-Instruct
    model_name: vinai/PhoGPT-7B5-Instruct
    tokenizer_name: vinai/PhoGPT-7B5-Instruct
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vinai/PhoGPT-7B5-Instruct

  - name: vinai/PhoGPT-4B-Chat
    model_name: vinai/PhoGPT-4B-Chat
    tokenizer_name: vinai/PhoGPT-4B-Chat
    max_sequence_length: 8192
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: vinai/PhoGPT-4B-Chat

  - name: huggingface/Gemma-3-Gaia-PT-BR-4b-it
    model_name: CEIA-UFG/Gemma-3-Gaia-PT-BR-4b-it
    tokenizer_name: CEIA-UFG/Gemma-3-Gaia-PT-BR-4b-it
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: CEIA-UFG/Gemma-3-Gaia-PT-BR-4b-it

  - name: recogna-nlp/bode-13b-alpaca-pt-br-no-peft
    model_name: recogna-nlp/bode-13b-alpaca-pt-br-no-peft
    tokenizer_name: recogna-nlp/bode-13b-alpaca-pt-br-no-peft
    max_sequence_length: 4094
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: recogna-nlp/bode-13b-alpaca-pt-br-no-peft

  - name: 22h/cabrita_7b_pt_850000
    model_name: 22h/cabrita_7b_pt_850000
    tokenizer_name: 22h/cabrita_7b_pt_850000
    max_sequence_length: 4094
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: 22h/cabrita_7b_pt_850000

  - name: PORTULAN/gervasio-7b-portuguese-ptbr-decoder
    model_name: PORTULAN/gervasio-7b-portuguese-ptbr-decoder
    tokenizer_name: PORTULAN/gervasio-7b-portuguese-ptbr-decoder
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: PORTULAN/gervasio-7b-portuguese-ptbr-decoder

  - name: TucanoBR/Tucano-2b4
    model_name: TucanoBR/Tucano-2b4
    tokenizer_name: TucanoBR/Tucano-2b4
    max_sequence_length: 4096
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: TucanoBR/Tucano-2b4

  - name: nicholasKluge/TeenyTinyLlama-460m
    model_name: nicholasKluge/TeenyTinyLlama-460m
    tokenizer_name: nicholasKluge/TeenyTinyLlama-460m
    max_sequence_length: 2048
    client_spec:
      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
      args:
        pretrained_model_name_or_path: nicholasKluge/TeenyTinyLlama-460m

  - name: openrouter/mistral-medium-3.1
    model_name: mistralai/mistral-medium-3.1
    tokenizer_name: mistralai/Mistral-7B-v0.1
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.openrouter_client.OpenRouterClient"
      args:
        model_name: mistralai/mistral-medium-3.1

  # DSPy Models (EXPERIMENTAL)
  # The following model configurations use the DSPyClient for inference with DSPy modules.
  
  - name: anthropic/claude-3-7-sonnet-20250219-dspy-zs-predict
    model_name: anthropic/claude-3-7-sonnet-20250219-dspy-zs-predict
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: Predict
        dspy_api_model: anthropic/claude-3-7-sonnet-20250219
        dspy_api_base: null

  - name: google/gemini-2.0-flash-001-dspy-zs-predict
    model_name: google/gemini-2.0-flash-001-dspy-zs-predict
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: Predict
        dspy_api_model: google/gemini-2.0-flash-001
        dspy_api_base: null

  - name: openai/gpt-4o-2024-05-13-dspy-zs-predict
    model_name: openai/gpt-4o-2024-05-13-dspy-zs-predict
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: Predict
        dspy_api_model: openai/gpt-4o-2024-05-13
        dspy_api_base: null

  - name: openai/o3-mini-2025-01-31-dspy-zs-predict
    model_name: openai/o3-mini-2025-01-31-dspy-zs-predict
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: Predict
        dspy_api_model: openai/o3-mini-2025-01-31
        dspy_api_base: null

  - name: anthropic/claude-3-7-sonnet-20250219-dspy-zs-cot
    model_name: anthropic/claude-3-7-sonnet-20250219-dspy-zs-cot
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: ChainOfThought
        dspy_api_model: anthropic/claude-3-7-sonnet-20250219
        dspy_api_base: null

  - name: google/gemini-2.0-flash-001-dspy-zs-cot
    model_name: google/gemini-2.0-flash-001-dspy-zs-cot
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: ChainOfThought
        dspy_api_model: google/gemini-2.0-flash-001
        dspy_api_base: null

  - name: openai/gpt-4o-2024-05-13-dspy-zs-cot
    model_name: openai/gpt-4o-2024-05-13-dspy-zs-cot
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: ChainOfThought
        dspy_api_model: openai/gpt-4o-2024-05-13
        dspy_api_base: null

  - name: openai/o3-mini-2025-01-31-dspy-zs-cot
    model_name: openai/o3-mini-2025-01-31-dspy-zs-cot
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: null
        dspy_module: ChainOfThought
        dspy_api_model: openai/o3-mini-2025-01-31
        dspy_api_base: null

  - name: anthropic/claude-3-7-sonnet-20250219-dspy-fs-bfrs
    model_name: anthropic/claude-3-7-sonnet-20250219-dspy-fs-bfrs
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/claude-3.7-sonnet/BootstrapFewShotWithRandomSearch.json
        dspy_module: ChainOfThought
        dspy_api_model: anthropic/claude-3-7-sonnet-20250219
        dspy_api_base: null

  - name: google/gemini-2.0-flash-001-dspy-fs-bfrs
    model_name: google/gemini-2.0-flash-001-dspy-fs-bfrs
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/gemini-2.0-flash/BootstrapFewShotWithRandomSearch.json
        dspy_module: ChainOfThought
        dspy_api_model: google/gemini-2.0-flash-001
        dspy_api_base: null

  - name: openai/gpt-4o-2024-05-13-dspy-fs-bfrs
    model_name: openai/gpt-4o-2024-05-13-dspy-fs-bfrs
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/gpt-4o/BootstrapFewShotWithRandomSearch.json
        dspy_module: ChainOfThought
        dspy_api_model: openai/gpt-4o-2024-05-13
        dspy_api_base: null

  - name: openai/o3-mini-2025-01-31-dspy-fs-bfrs
    model_name: openai/o3-mini-2025-01-31-dspy-fs-bfrs
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/o3-mini/BootstrapFewShotWithRandomSearch.json
        dspy_module: ChainOfThought
        dspy_api_model: openai/o3-mini-2025-01-31
        dspy_api_base: null

  - name: anthropic/claude-3-7-sonnet-20250219-dspy-fs-miprov2
    model_name: anthropic/claude-3-7-sonnet-20250219-dspy-fs-miprov2
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/claude-3.7-sonnet/MIPROv2.json
        dspy_module: ChainOfThought
        dspy_api_model: anthropic/claude-3-7-sonnet-20250219
        dspy_api_base: null

  - name: google/gemini-2.0-flash-001-dspy-fs-miprov2
    model_name: google/gemini-2.0-flash-001-dspy-fs-miprov2
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/gemini-2.0-flash/MIPROv2.json
        dspy_module: ChainOfThought
        dspy_api_model: google/gemini-2.0-flash-001
        dspy_api_base: null

  - name: openai/gpt-4o-2024-05-13-dspy-fs-miprov2
    model_name: openai/gpt-4o-2024-05-13-dspy-fs-miprov2
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/gpt-4o/MIPROv2.json
        dspy_module: ChainOfThought
        dspy_api_model: openai/gpt-4o-2024-05-13
        dspy_api_base: null

  - name: openai/o3-mini-2025-01-31-dspy-fs-miprov2
    model_name: openai/o3-mini-2025-01-31-dspy-fs-miprov2
    tokenizer_name: openai/o200k_base
    max_sequence_length: 128000
    client_spec:
      class_name: "helm.clients.dspy_client.DSPyClient"
      args:
        dspy_agent_url: https://raw.githubusercontent.com/StanfordMIMI/dspy-helm/cecc4e742ec0d342d6349c80625afdd238df7fd4/agents/{scenario}/o3-mini/MIPROv2.json
        dspy_module: ChainOfThought
        dspy_api_model: openai/o3-mini-2025-01-31
        dspy_api_base: null
