---
############################################################
perturbations: []
############################################################
metrics:
  - name: num_references
    display_name: '# ref'
    description: Number of references.
  - name: num_train_trials
    display_name: '# trials'
    description: Number of trials, where in each trial we choose an independent, random set of training instances.
  - name: estimated_num_tokens_cost
    display_name: 'cost'
    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
  - name: num_prompt_tokens
    display_name: '# prompt tokens'
    description: Number of tokens in the prompt.
  - name: num_prompt_characters
    display_name: '# prompt chars'
    description: Number of characters in the prompt.
  - name: num_completion_tokens
    display_name: '# completion tokens'
    description: Actual number of completion tokens (over all completions).
  - name: num_output_tokens
    display_name: '# output tokens'
    description: Actual number of output tokens.
  - name: max_num_output_tokens
    display_name: 'Max output tokens'
    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
  - name: num_requests
    display_name: '# requests'
    description: Number of distinct API requests.
  - name: num_instances
    display_name: '# eval'
    description: Number of evaluation instances.
  - name: num_train_instances
    display_name: '# train'
    description: Number of training instances (e.g., in-context examples).
  - name: prompt_truncated
    display_name: truncated
    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
  - name: finish_reason_length
    display_name: finish b/c length
    description: Fraction of instances where the the output was terminated because of the max tokens limit.
  - name: finish_reason_stop
    display_name: finish b/c stop
    description: Fraction of instances where the the output was terminated because of the stop sequences.
  - name: finish_reason_endoftext
    display_name: finish b/c endoftext
    description: Fraction of instances where the the output was terminated because the end of text token was generated.
  - name: finish_reason_unknown
    display_name: finish b/c unknown
    description: Fraction of instances where the the output was terminated for unknown reasons.
  - name: num_completions
    display_name: '# completions'
    description: Number of completions.
  - name: predicted_index
    display_name: Predicted index
    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).

  # Vision Language metrics [image]:
  - name: earth_mover_similarity
    display_name: Earth Mover Similarity
    short_display_name: EMD-Sim
    description: 1 - Earth Mover Distance [(Rubner and Tomasi, 2000)](https://www.cs.cmu.edu/~efros/courses/LBMV07/Papers/rubner-jcviu-00.pdf) between an image generated by the model and the target image.
    lower_is_better: false
  - name: pixel_similarity
    display_name: Pixel Similarity
    short_display_name: PS
    description: Pixel Similarity between an image generated by the model and the target image.
    lower_is_better: false
  - name: sift_similarity
    display_name: SIFT Similarity
    short_display_name: SIFT
    description: SIFT Similarity (Scale-Invariant Feature Transform) [(Lowe, 1999)](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=790410) between an image generated by the model and the target image.
    lower_is_better: false
  - name: compilation_success
    display_name: Compilation success
    description: Fraction of instances where the generated code compiles successfully.
    lower_is_better: false
  - name: lpips_similarity
    display_name: LPIPS similarity
    short_display_name: LPIPS
    description: LPIPS similarity (Learned Perceptual Image Patch Similarity) [(Zhang et al., 2018)](https://arxiv.org/abs/1801.03924) between an image generated by the model and the target image.
    lower_is_better: false
  - name: fid_similarity
    display_name: FID similarity
    short_display_name: FID
    description: FID similarity (Fréchet Inception Distance) [(Heusel et al., 2017)](https://arxiv.org/abs/1706.08500) between an image generated by the model and the target image.
    lower_is_better: false
  - name: ssim_similarity
    display_name: SSIM
    short_display_name: SSIM
    description: SSIM similarity (Structural Similarity Index) [(Wang et al., 2004)](https://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf) between an image generated by the model and the target image.
    lower_is_better: false

  # Accuracy metrics:
  - name: exact_match
    display_name: Exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference exactly.
    lower_is_better: false
  - name: quasi_exact_match
    display_name: Quasi-exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
    lower_is_better: false
  - name: quasi_leave_articles_exact_match
    display_name: Quasi-exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
    lower_is_better: false
  - name: prefix_exact_match
    display_name: Prefix exact match
    short_display_name: PEM
    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
    lower_is_better: false
  - name: quasi_prefix_exact_match
    # TODO: should call this prefix_quasi_exact_match
    display_name: Prefix quasi-exact match
    short_display_name: PEM
    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
    lower_is_better: false

  - name: rouge_1
    display_name: ROUGE-1
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
    lower_is_better: false
  - name: rouge_2
    display_name: ROUGE-2
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
    lower_is_better: false
  - name: rouge_l
    display_name: ROUGE-L
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
    lower_is_better: false
  - name: bleu_1
    display_name: BLEU-1
    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 1-gram overlap.
    lower_is_better: false
  - name: bleu_4
    display_name: BLEU-4
    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 4-gram overlap.
    lower_is_better: false
  - name: f1_score
    display_name: F1
    description: Average F1 score in terms of word overlap between the model output and correct reference.
    lower_is_better: false
  - name: cider
    display_name: CIDEr
    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
    lower_is_better: false
  - name: prometheus_vision
    display_name: Prometheus Vision rating
    description: Scores are from 1 to 5, where 5 is the best.
    lower_is_better: false

  # Toxicity metrics
  - name: expected_max_toxicity
    display_name: Average max toxicity
    short_display_name: Avg. max toxicity
    lower_is_better: true
    description: Average max toxicity score across all completions (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
  - name: max_toxicity_probability
    display_name: max toxicity
    lower_is_better: true
    description: Fraction of instances where most toxic completion is above 0.5 toxicity (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
  - name: toxic_frac
    display_name: Toxic fraction
    lower_is_better: true
    description: Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity classifier).

############################################################
metric_groups:
  - name: accuracy
    display_name: Accuracy
    metrics:
      - name: ${main_name}
        split: ${main_split}

  - name: general_information
    display_name: General information
    metrics:
    - name: num_instances
      split: ${main_split}
    - name: num_train_instances
      split: ${main_split}
    - name: prompt_truncated
      split: ${main_split}
    - name: num_prompt_tokens
      split: ${main_split}
    - name: num_output_tokens
      split: ${main_split}


############################################################
run_groups:
  - name: core_scenarios
    display_name: All
    description: All scenarios across capabilities
    category: All scenarios
    subgroups:
      - msr_vtt

  - name: msr_vtt
    display_name: MST-VTT
    description: A large-scale video benchmark for video understanding, especially the emerging task of translating video to text.
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: f1_score
      main_split: test
    taxonomy:
      task: captioning
      what: Real-world videos
      who: Human experts
      when: "2016"
      language: English
