############################################################
metrics:
  # Infrastructure metrics:
  - name: num_perplexity_tokens
    display_name: '# tokens'
    description: Average number of tokens in the predicted output (for language modeling, the input too).
  - name: num_bytes
    display_name: '# bytes'
    description: Average number of bytes in the predicted output (for language modeling, the input too).

  - name: num_references
    display_name: '# ref'
    description: Number of references.
  - name: num_train_trials
    display_name: '# trials'
    description: Number of trials, where in each trial we choose an independent, random set of training instances.
  - name: estimated_num_tokens_cost
    display_name: 'cost'
    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
  - name: num_prompt_tokens
    display_name: '# prompt tokens'
    description: Number of tokens in the prompt.
  - name: num_prompt_characters
    display_name: '# prompt chars'
    description: Number of characters in the prompt.
  - name: num_completion_tokens
    display_name: '# completion tokens'
    description: Actual number of completion tokens (over all completions).
  - name: num_output_tokens
    display_name: '# output tokens'
    description: Actual number of output tokens.
  - name: max_num_output_tokens
    display_name: 'Max output tokens'
    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
  - name: num_requests
    display_name: '# requests'
    description: Number of distinct API requests.
  - name: num_instances
    display_name: '# eval'
    description: Number of evaluation instances.
  - name: num_train_instances
    display_name: '# train'
    description: Number of training instances (e.g., in-context examples).
  - name: prompt_truncated
    display_name: truncated
    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
  - name: finish_reason_length
    display_name: finish b/c length
    description: Fraction of instances where the the output was terminated because of the max tokens limit.
  - name: finish_reason_stop
    display_name: finish b/c stop
    description: Fraction of instances where the the output was terminated because of the stop sequences.
  - name: finish_reason_endoftext
    display_name: finish b/c endoftext
    description: Fraction of instances where the the output was terminated because the end of text token was generated.
  - name: finish_reason_unknown
    display_name: finish b/c unknown
    description: Fraction of instances where the the output was terminated for unknown reasons.
  - name: num_completions
    display_name: '# completions'
    description: Number of completions.
  - name: predicted_index
    display_name: Predicted index
    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).

  # Accuracy metrics:
  - name: exact_match
    display_name: Exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference exactly.
    lower_is_better: false
  - name: classification_macro_f1
    display_name: Macro-F1
    description: Population-level macro-averaged F1 score.
    lower_is_better: false
  - name: classification_micro_f1
    display_name: Micro-F1
    description: Population-level micro-averaged F1 score.
    lower_is_better: false
  - name: wer_score
    display_name: Word Error Rate
    description: Transcription error rate.
    lower_is_better: true
  - name: mer_score
    display_name: Character Error Rate
    description: Character error rate.
    lower_is_better: true

############################################################
perturbations: []

############################################################
metric_groups:
  - name: accuracy
    display_name: Accuracy
    hide_win_rates: true
    metrics:
      - name: exact_match
        split: ${main_split}
      - name: classification_macro_f1
        split: ${main_split}
      - name: classification_micro_f1
        split: ${main_split}

  - name: transcription_accuracy
    display_name: Transcription Accuracy
    hide_win_rates: true
    metrics:
      - name: wer_score
        split: ${main_split}
      - name: mer_score
        split: ${main_split}

  - name: efficiency
    display_name: Efficiency
    metrics:
    - name: inference_runtime
      split: ${main_split}

  - name: general_information
    display_name: General information
    hide_win_rates: true
    metrics:
    - name: num_instances
      split: ${main_split}
    - name: num_train_instances
      split: ${main_split}
    - name: prompt_truncated
      split: ${main_split}
    - name: num_prompt_tokens
      split: ${main_split}
    - name: num_output_tokens
      split: ${main_split}

############################################################

run_groups:
  - name: slp
    display_name: SLP Scenarios
    description: SLP-language scenarios
    category: All scenarios
    subgroups:
      - disorder_diagnosis
      - transcription
      - symptom_diagnosis
      - disorder_type_diagnosis


  - name: disorder_diagnosis
    display_name: Disorder Diagnosis Accuracy
    description: >
      Macro-averaged accuracy on disorder diagnosis for pediatric speech disorder.
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: classification_micro_f1
      main_split: test
    taxonomy:
      task: classification
      what: n/a
      who: n/a
      when: "?"
      language: English

  - name: transcription
    display_name: Transcription Accuracy
    description: >
      Model transcription accuracy on understanding disordered pediatric speech
    metric_groups:
      - transcription_accuracy
      - efficiency
      - general_information
    environment:
      main_name: wer_score
      main_split: test
    taxonomy:
      task: transcription
      what: disordered pediatric speech
      who: n/a
      when: "?"
      language: English

  - name: symptom_diagnosis
    display_name: Symptom Diagnosis Accuracy
    description: >
      Macro-averaged accuracy on symptom diagnosis for pediatric speech disorder.
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: classification_micro_f1
      main_split: test
    taxonomy:
      task: classification
      what: n/a
      who: n/a
      when: "?"
      language: English

  - name: disorder_type_diagnosis
    display_name: Disorder Type Diagnosis Accuracy
    description: >
      Macro-averaged accuracy on disorder type diagnosis for pediatric speech disorder.
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: classification_micro_f1
      main_split: test
    taxonomy:
      task: classification
      what: n/a
      who: n/a
      when: "?"
      language: English