name: "{{ model_name }}"
backend: "python"
max_batch_size: 8

# Hugging face model path. Parameters must follow this
# key/value structure
parameters: {
  key: "pretrained_model_name_or_path",
  value: {string_value: "{{ pretrained_model_name_or_path | default("rayliuca/coedit-large-onnx") }}"}
}

parameters: {
  key: "grammared_language_model_config",
  value: {string_value: "{{ json_model_config }}"}
}

# Triton should expect as input a single string of set
# length named 'text_input'
input [
  {
    name: "text_input"
    data_type: TYPE_STRING
    dims: [ 1 ]
  }
]

# Triton should expect to respond with a single string
# output of variable length named 'text_output'
output [
  {
    name: "text_output"
    data_type: TYPE_STRING
    dims: [ -1 ]
  }
]

# Instance groups - configure based on available resources
instance_group [
  {
    count: 1
    # kind: KIND_GPU
    # gpus: [0]
    kind: KIND_CPU
  }
]


# Python backend requires these parameters
parameters: {
  key: "EXECUTION_ENV_PATH",
  value: {string_value: "/opt/tritonserver/python_backend_env"}
}

# Dynamic batching for better throughput
dynamic_batching {
  preferred_batch_size: [1, 2, 4, 8]
  max_queue_delay_microseconds: 750
}