---
############################################################
# For backwards compatibility with older versions of HELM.
# TODO: Remove this after 2024-09-01.
adapter: []
############################################################
metrics:
  # Infrastructure metrics:
  - name: num_perplexity_tokens
    display_name: '# tokens'
    description: Average number of tokens in the predicted output (for language modeling, the input too).
  - name: num_bytes
    display_name: '# bytes'
    description: Average number of bytes in the predicted output (for language modeling, the input too).

  - name: num_references
    display_name: '# ref'
    description: Number of references.
  - name: num_train_trials
    display_name: '# trials'
    description: Number of trials, where in each trial we choose an independent, random set of training instances.
  - name: estimated_num_tokens_cost
    display_name: 'cost'
    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
  - name: num_prompt_tokens
    display_name: '# prompt tokens'
    description: Number of tokens in the prompt.
  - name: num_prompt_characters
    display_name: '# prompt chars'
    description: Number of characters in the prompt.
  - name: num_completion_tokens
    display_name: '# completion tokens'
    description: Actual number of completion tokens (over all completions).
  - name: num_output_tokens
    display_name: '# output tokens'
    description: Actual number of output tokens.
  - name: max_num_output_tokens
    display_name: 'Max output tokens'
    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
  - name: num_requests
    display_name: '# requests'
    description: Number of distinct API requests.
  - name: num_instances
    display_name: '# eval'
    description: Number of evaluation instances.
  - name: num_train_instances
    display_name: '# train'
    description: Number of training instances (e.g., in-context examples).
  - name: prompt_truncated
    display_name: truncated
    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
  - name: finish_reason_length
    display_name: finish b/c length
    description: Fraction of instances where the the output was terminated because of the max tokens limit.
  - name: finish_reason_stop
    display_name: finish b/c stop
    description: Fraction of instances where the the output was terminated because of the stop sequences.
  - name: finish_reason_endoftext
    display_name: finish b/c endoftext
    description: Fraction of instances where the the output was terminated because the end of text token was generated.
  - name: finish_reason_unknown
    display_name: finish b/c unknown
    description: Fraction of instances where the the output was terminated for unknown reasons.
  - name: num_completions
    display_name: '# completions'
    description: Number of completions.
  - name: predicted_index
    display_name: Predicted index
    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).

  # Vision Language metrics [text]:
  - name: edit_similarity
    display_name: Edit similarity (Levenshtein)
    short_display_name: Edit sim.
    lower_is_better: false
    description: Average Levenshtein edit similarity (1 - distance normalized by length of longer sequence) between model generation and reference.

  # Vision Language metrics [image]:
  - name: earth_mover_similarity
    display_name: Earth Mover Similarity
    short_display_name: EMD-Sim
    description: 1 - Earth Mover Distance [(Rubner and Tomasi, 2000)](https://www.cs.cmu.edu/~efros/courses/LBMV07/Papers/rubner-jcviu-00.pdf) between an image generated by the model and the target image.
    lower_is_better: false
  - name: pixel_similarity
    display_name: Pixel Similarity
    short_display_name: PS
    description: Pixel Similarity between an image generated by the model and the target image.
    lower_is_better: false
  - name: sift_similarity
    display_name: SIFT Similarity
    short_display_name: SIFT
    description: SIFT Similarity (Scale-Invariant Feature Transform) [(Lowe, 1999)](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=790410) between an image generated by the model and the target image.
    lower_is_better: false
  - name: compilation_success
    display_name: Compilation success
    description: Fraction of instances where the generated code compiles successfully.
    lower_is_better: false
  - name: lpips_similarity
    display_name: LPIPS similarity
    short_display_name: LPIPS
    description: LPIPS similarity (Learned Perceptual Image Patch Similarity) [(Zhang et al., 2018)](https://arxiv.org/abs/1801.03924) between an image generated by the model and the target image.
    lower_is_better: false
  - name: fid_similarity
    display_name: FID similarity
    short_display_name: FID
    description: FID similarity (Fréchet Inception Distance) [(Heusel et al., 2017)](https://arxiv.org/abs/1706.08500) between an image generated by the model and the target image.
    lower_is_better: false
  - name: ssim_similarity
    display_name: SSIM
    short_display_name: SSIM
    description: SSIM similarity (Structural Similarity Index) [(Wang et al., 2004)](https://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf) between an image generated by the model and the target image.
    lower_is_better: false

  # Accuracy metrics:
  - name: exact_match
    display_name: Exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference exactly.
    lower_is_better: false
  - name: quasi_exact_match
    display_name: Quasi-exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
    lower_is_better: false
  - name: quasi_leave_articles_exact_match
    display_name: Quasi-exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
    lower_is_better: false
  - name: prefix_exact_match
    display_name: Prefix exact match
    short_display_name: PEM
    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
    lower_is_better: false
  - name: quasi_prefix_exact_match
    # TODO: should call this prefix_quasi_exact_match
    display_name: Prefix quasi-exact match
    short_display_name: PEM
    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
    lower_is_better: false

  - name: rouge_1
    display_name: ROUGE-1
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
    lower_is_better: false
  - name: rouge_2
    display_name: ROUGE-2
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
    lower_is_better: false
  - name: rouge_l
    display_name: ROUGE-L
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
    lower_is_better: false
  - name: bleu_1
    display_name: BLEU-1
    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 1-gram overlap.
    lower_is_better: false
  - name: bleu_4
    display_name: BLEU-4
    description: Average BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 4-gram overlap.
    lower_is_better: false
  - name: f1_score
    display_name: F1
    description: Average F1 score in terms of word overlap between the model output and correct reference.
    lower_is_better: false
  - name: cider
    display_name: CIDEr
    description: Evaluates the quality of generated caption by measuring the weighted similarity of n-grams between the captions and a set of human-written reference captions, emphasizing informativeness and consensus.
    lower_is_better: false
  - name: prometheus_vision
    display_name: Prometheus Vision rating
    description: Scores are from 1 to 5, where 5 is the best.
    lower_is_better: false

  # Toxicity metrics
  - name: expected_max_toxicity
    display_name: Average max toxicity
    short_display_name: Avg. max toxicity
    lower_is_better: true
    description: Average max toxicity score across all completions (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
  - name: max_toxicity_probability
    display_name: max toxicity
    lower_is_better: true
    description: Fraction of instances where most toxic completion is above 0.5 toxicity (based on the PerspectiveAPI toxicity classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).
  - name: toxic_frac
    display_name: Toxic fraction
    lower_is_better: true
    description: Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity classifier).

############################################################
perturbations:
  - name: translate
    display_name: Translate
    description: Translate prompts to different languages.
  - name: robustness
    display_name: Robustness
    description: Computes worst case over different robustness perturbations (misspellings, formatting, contrast sets).
  - name: fairness
    display_name: Fairness
    description: Computes worst case over different fairness perturbations (changing dialect, race of names, gender).
  - name: typos
    display_name: Typos
    description: >
      Randomly adds typos to each token in the input with probability 0.05 and computes the per-instance worst-case
      performance between perturbed and unperturbed versions.
  - name: synonym
    display_name: Synonyms
    description: >
      Randomly substitutes words in the input with WordNet synonyms with probability 0.5 and computes the per-instance
      worst-case performance between perturbed and unperturbed versions.
  - name: dialect
    display_name: SAE -> AAE
    short_display_name: Dialect
    description: >
      Deterministically substitutes SAE words in input with AAE counterparts using validated dictionary of [Ziems et al. (2022)](https://aclanthology.org/2022.acl-long.258/) and computes the per-instance worst-case performance between perturbed and unperturbed versions.
  - name: race
    display_name: First names by race (White -> Black)
    short_display_name: Race
    description: >
      Deterministically substitutes White first names with Black first names sampled from the lists of [Caliskan et al. (2017)](https://www.science.org/doi/10.1126/science.aal4230) and computes the per-instance worst-case performance between perturbed and unperturbed versions.
  - name: gender
    display_name: Pronouns by gender (Male -> Female)
    short_display_name: Gender
    description: >
      Deterministically substitutes male pronouns with female pronouns and computes the per-instance worst-case
      performance between perturbed and unperturbed versions.

############################################################
metric_groups:
  - name: accuracy
    display_name: Accuracy
    metrics:
      - name: ${main_name}
        split: ${main_split}

  - name: general_information
    display_name: General information
    metrics:
    - name: num_instances
      split: ${main_split}
    - name: num_train_instances
      split: ${main_split}
    - name: prompt_truncated
      split: ${main_split}
    - name: num_prompt_tokens
      split: ${main_split}
    - name: num_output_tokens
      split: ${main_split}

  - name: toxicity
    display_name: Toxicity
    metrics:
      - name: toxic_frac
        split: ${main_split}

  - name: fairness
    display_name: Fairness
    metrics:
      - name: ${main_name}
        split: ${main_split}
        perturbation_name: fairness

  - name: robustness
    display_name: Robustness
    metrics:
      - name: ${main_name}
        split: ${main_split}
        perturbation_name: robustness

  - name: translate
    display_name: Translate
    metrics:
      - name: ${main_name}
        split: ${main_split}
        perturbation_name: translate


############################################################
run_groups:
  - name: core_scenarios
    display_name: All
    description: All scenarios across capabilities
    category: All scenarios
    subgroups:
      - visual_perception
      - reasoning
      - knowledge
      - bias
      - fairness
      - safety
      - toxicity
      - robustness
      - multilinguality
  - name: visual_perception
    display_name: Visual perception
    description: Is the output semantically correct, given the text and image inputs?
    category: Core scenarios
    subgroups:
      - vqa_base
      - viz_wiz
      - flickr30k
      - pope
      - mm_star_perception
      - blink_perception
  - name: reasoning
    display_name: Reasoning
    description: Does the model understand objects, counts and spatial relations? Can the model reason about both the text and image input?
    category: Core scenarios
    subgroups:
      - mmmu
      - exams_v
      - gqa
      - math_vista
      - seed_bench
      - mementos
      - real_world_qa
      - mm_star_reasoning
      - blink_reasoning
  - name: knowledge
    display_name: Knowledge
    description: Does the model have knowledge about the world and common sense?
    category: Core scenarios
    subgroups:
      - a_okvqa_base
      - mme
      - vibe_eval
      - mm_star_knowledge
      - blink_knowledge
  - name: bias
    display_name: Bias
    description: Are the generations biased in demographic representation? We focus on gender and skin tone bias.
    category: Core scenarios
    subgroups:
      - pairs
  - name: fairness
    display_name: Fairness
    description: Does the model exhibit performance disparities across different groups? We focus on gender, dialect and geographic bias.
    category: Core scenarios
    subgroups:
      - vqa_dialect
      - a_okvqa_dialect
      - crossmodal_3600
      - fair_face
      - bingo_fairness
  - name: toxicity
    display_name: Toxicity
    description: Does the model generate toxic or inappropriate content? Can the model identify toxic or inappropriate content?
    category: Core scenarios
    subgroups:
      - hateful_memes
  - name: safety
    display_name: Safety
    description: Refusing to produce answers that cause harm to humans
    category: Core scenarios
    subgroups:
      - mm_safety_bench
  - name: robustness
    display_name: Robustness
    description: Is the model robust to perturbations? We focus on both text and image perturbations.
    category: Core scenarios
    subgroups:
      - unicorn
      - bingo
  - name: multilinguality
    display_name: Multilinguality
    description: Do the model support non-English languages?
    category: Core scenarios
    subgroups:
      - a_okvqa_chinese
      - a_okvqa_hindi
      - a_okvqa_spanish
      - a_okvqa_swahili
      - bingo_multilinguality
  - name: a_okvqa_base
    display_name: A-OKVQA
    description: A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense and world knowledge to answer ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: English

  - name: a_okvqa_dialect
    display_name: A-OKVQA (AAE)
    description: African-American English Perturbation + A-OKVQA ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718)).
    metric_groups:
      - fairness
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: English

  - name: a_okvqa_chinese
    display_name: A-OKVQA (chinese)
    description: Chinese Translation Perturbation + A-OKVQA ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718)).
    metric_groups:
      - translate
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: Chinese

  - name: a_okvqa_hindi
    display_name: A-OKVQA (hindi)
    description: Hindi Translation Perturbation + A-OKVQA ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718)).
    metric_groups:
      - translate
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: Hindi

  - name: a_okvqa_spanish
    display_name: A-OKVQA (spanish)
    description: Spanish Translation Perturbation + A-OKVQA ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718)).
    metric_groups:
      - translate
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: Spanish

  - name: a_okvqa_swahili
    display_name: A-OKVQA (swahili)
    description: Swahili Translation Perturbation + A-OKVQA ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718)).
    metric_groups:
      - translate
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: Swahili

  - name: mm_star_perception
    display_name: MM-Star (Perception subsets)
    description: MM-STAR is an elite vision-indispensable multi-modal benchmark comprising 1,500 challenge samples meticulously selected by humans. ([Chen et al., 2024](https://arxiv.org/abs/2403.20330)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2024"
      language: English

  - name: mm_star_reasoning
    display_name: MM-Star (Reasoning subsets)
    description: MM-STAR is an elite vision-indispensable multi-modal benchmark comprising 1,500 challenge samples meticulously selected by humans. ([Chen et al., 2024](https://arxiv.org/abs/2403.20330)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2024"
      language: English

  - name: mm_star_knowledge
    display_name: MM-Star (Knowledge subsets)
    description: MM-STAR is an elite vision-indispensable multi-modal benchmark comprising 1,500 challenge samples meticulously selected by humans. ([Chen et al., 2024](https://arxiv.org/abs/2403.20330)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2024"
      language: English

  - name: blink_perception
    display_name: BLINK (Perception subsets)
    description: BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans within a blink, but difficulty for VLMs. ([Fu, 2024](https://arxiv.org/abs/2404.12390)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2024"
      language: English

  - name: blink_knowledge
    display_name: BLINK (Knowledge subsets)
    description: BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans within a blink, but difficulty for VLMs. ([Fu, 2024](https://arxiv.org/abs/2404.12390)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2024"
      language: English

  - name: blink_reasoning
    display_name: BLINK (Reasoning subsets)
    description: BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans within a blink, but difficulty for VLMs. ([Fu, 2024](https://arxiv.org/abs/2404.12390)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2024"
      language: English

  - name: crossmodal_3600
    display_name: Crossmodal 3600
    description: Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated with human-generated reference captions in 36 languages. ([Thapliyal et al., 2022](https://arxiv.org/abs/2205.12522))
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: multilingual captioning
      what: Real-world images
      who: Human experts
      when: "2022"
      language: 36 languages

  - name: flickr30k
    display_name: Flickr30k
    description: An image caption corpus consisting of 158,915 crowd-sourced captions describing 31,783 Flickr images. ([Young et al., 2014](https://shannon.cs.illinois.edu/DenotationGraph/TACLDenotationGraph.pdf))
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: image captioning
      what: Flickr images
      who: Human experts
      when: "2014"
      language: English

  - name: gqa
    display_name: GQA
    description: Questions about real-world visual reasoning and compositional QA ([Hudson and Manning, 2019](https://arxiv.org/abs/1902.09506)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: short-answer question answering
      what: Real-world images
      who: Human experts
      when: "2019"
      language: English

  - name: hateful_memes
    display_name: Hateful Memes
    description: Multimodal hate speech detection in memes ([Kiela et al., 2020](https://arxiv.org/abs/2005.04790)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: toxicity identification
      what: Memes
      who: Human experts
      when: "2020"
      language: English

  - name: mm_safety_bench
    display_name: MM-SafetyBench
    description: Exposes the vulnerability of open-source VLMs with toxic and biased content ([Liu et al., 2023](https://arxiv.org/abs/2311.17600)).
    metric_groups:
      - accuracy
      - toxicity
      - general_information
    environment:
      main_name: toxic_frac
      main_split: test
    taxonomy:
      task: toxicity mitigation
      what: Jail-break images
      who: Human experts
      when: "2023"
      language: English

  - name: viz_wiz
    display_name: VizWiz
    description: A benchmark for visual question answering with images and questions created by visually impaired people ([Gurari et al., 2018](https://arxiv.org/abs/1802.08218)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: short-answer question answering
      what: Real-world images
      who: Visually impaired people
      when: "2018"
      language: English

  - name: vqa_base
    display_name: VQAv2
    description: Open-ended questions about real-world images ([Goyal et al., 2017](https://arxiv.org/abs/1612.00837)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: short-answer question answering
      what: Real-world images
      who: Human experts
      when: "2017"
      language: English

  - name: vqa_dialect
    display_name: VQAv2 (AAE)
    description: African-American English Perturbation + Open-ended questions about real-world images ([Goyal et al., 2017](https://arxiv.org/abs/1612.00837)).
    metric_groups:
      - fairness
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: short-answer question answering
      what: Real-world images
      who: Human experts
      when: "2017"
      language: English

  - name: math_vista
    display_name: MathVista
    description: A benchmark designed to combine challenges from diverse mathematical and visual tasks ([Lu et al., 2024](https://arxiv.org/abs/2310.02255)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: exact_match
      main_split: test
    taxonomy:
      task: multiple-choice question answering
      what: Evaluating Math Reasoning in Visual Contexts
      who: Human experts
      when: "2024"
      language: English

  - name: mmmu
    display_name: MMMU
    description: A benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning ([Yue et al., 2023](https://arxiv.org/abs/2311.16502)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering
      who: Human experts
      when: "2023"
      language: English
  
  - name: unicorn
    display_name: Unicorn
    description: Safety Evaluation Benchmark for Evaluating on Out-of-Distribution and Sketch Images ([Tu et al., 2023](https://arxiv.org/abs/2311.16101)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: OOD images and sketch images
      who: Human experts
      when: "2023"
      language: English
  
  - name: bingo
    display_name: Bingo
    description: Open-ended questions about biased images and hallucinations-inducing images ([Cui et al., 2023](https://arxiv.org/abs/2311.03287)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Biased images about Region, OCR, Factual, Text-to-Image and Image-to-Image inference challenges
      who: Human experts
      when: "2023"
      language: English, Chinese, Japanese, etc.

  - name: bingo_fairness
    display_name: Bingo (fairness)
    description: Open-ended questions about biased images and hallucinations-inducing images ([Cui et al., 2023](https://arxiv.org/abs/2311.03287)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Biased images about Region, OCR, Factual, Text-to-Image and Image-to-Image inference challenges
      who: Human experts
      when: "2023"
      language: English, Chinese, Japanese, etc.

  - name: bingo_multilinguality
    display_name: Bingo (multilinguality)
    description: Open-ended questions about biased images and hallucinations-inducing images ([Cui et al., 2023](https://arxiv.org/abs/2311.03287)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Biased images about Region, OCR, Factual, Text-to-Image and Image-to-Image inference challenges
      who: Human experts
      when: "2023"
      language: English, Chinese, Japanese, etc.

  - name: pope
    display_name: POPE
    description: Open-ended questions about object appearance in real-world images for evaluating hallucination behaviour ([Li et al., 2023](https://aclanthology.org/2023.emnlp-main.20)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: English

  - name: seed_bench
    display_name: Seed Bench 
    description: A massive multiple-choice question-answering benchmark that spans 9 evaluation aspects with the image input including the comprehension of both the image and video modality ([Li et al., 2023](https://arxiv.org/abs/2307.16125)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: English

  - name: mme
    display_name: MME
    description: A comprehensive MLLM Evaluation benchmark with perception and cognition evaluations on 14 subtasks ([Fu et al., 2023](https://arxiv.org/abs/2306.13394)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: multiple-choice question answering
      what: Real-world images
      who: Human experts
      when: "2023"
      language: English

  - name: vibe_eval
    display_name: Vibe Eval
    description: A difficult evaluation suite for measuring progress of multimodal language models with day-to-day tasks ([Padlewski et al., 2024](https://arxiv.org/abs/2405.02287)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Knowledge intensive
      who: Human experts
      when: "2024"
      language: English

  - name: mementos
    display_name: Mementos
    description: A Comprehensive Benchmark for Multimodal Large Language Model Reasoning over Image Sequences ([Wang et al., 2024](https://arxiv.org/abs/2401.10529)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: prometheus_vision
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Image sequences of comics, daily life and robotics
      who: Human experts
      when: "2024"
      language: English

  - name: pairs
    display_name: PAIRS
    description: Examining gender and racial bias using parallel images ([Fraser et al., 2024](https://arxiv.org/abs/2402.05779)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: multiple-choice question answering
      what: Bias
      who: Human experts
      when: "2024"
      language: English

  - name: fair_face
    display_name: FairFace
    description: Identify the race, gender or age of a photo of a person ([Karkkainen et al., 2019](https://arxiv.org/abs/1908.04913)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: valid
    taxonomy:
      task: multiple-choice question answering
      what: Fairness
      who: Human experts
      when: "2019"
      language: English

  - name: real_world_qa
    display_name: RealWorldQA
    description: A benchmark designed to to evaluate real-world spatial understanding capabilities of multimodal models ([xAI, 2024](https://x.ai/blog/grok-1.5v)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_leave_articles_exact_match
      main_split: test
    taxonomy:
      task: short-answer question answering
      what: Real world images
      who: Human experts
      when: "2024"
      language: English

  - name: exams_v
    display_name: Exams-V
    description: A multimodal and multilingual benchmark with knowledge-intensive exam questions covering natural science, social science, and other miscellaneous studies ([Das et al., 2024]( https://arxiv.org/abs/2403.10378)).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_prefix_exact_match
      main_split: test
    taxonomy:
      task: multiple-choice question answering
      what: Exam questions
      who: Human experts
      when: "2024"
      language: English, Chinese, Croation, Hungarian, Arabic, Serbian, Bulgarian, English, German, French, Spanish, Polish
