---
############################################################
metrics:

- name: num_perplexity_tokens
  display_name: "# tokens"
  description: Average number of tokens in the predicted output (for language modeling,
    the input too).

- name: num_bytes
  display_name: "# bytes"
  description: Average number of bytes in the predicted output (for language modeling,
    the input too).

- name: num_references
  display_name: "# ref"
  description: Number of references.

- name: num_train_trials
  display_name: "# trials"
  description: Number of trials, where in each trial we choose an independent, random
    set of training instances.

- name: estimated_num_tokens_cost
  display_name: cost
  description: An estimate of the number of tokens (including prompt and output completions)
    needed to perform the request.

- name: num_prompt_tokens
  display_name: "# prompt tokens"
  description: Number of tokens in the prompt.

- name: num_prompt_characters
  display_name: "# prompt chars"
  description: Number of characters in the prompt.

- name: num_completion_tokens
  display_name: "# completion tokens"
  description: Actual number of completion tokens (over all completions).

- name: num_output_tokens
  display_name: "# output tokens"
  description: Actual number of output tokens.

- name: max_num_output_tokens
  display_name: Max output tokens
  description: Maximum number of output tokens (overestimate since we might stop earlier
    due to stop sequences).

- name: num_requests
  display_name: "# requests"
  description: Number of distinct API requests.

- name: num_instances
  display_name: "# eval"
  description: Number of evaluation instances.

- name: num_train_instances
  display_name: "# train"
  description: Number of training instances (e.g., in-context examples).

- name: prompt_truncated
  display_name: truncated
  description: Fraction of instances where the prompt itself was truncated (implies
    that there were no in-context examples).

- name: finish_reason_length
  display_name: finish b/c length
  description: Fraction of instances where the the output was terminated because of
    the max tokens limit.

- name: finish_reason_stop
  display_name: finish b/c stop
  description: Fraction of instances where the the output was terminated because of
    the stop sequences.

- name: finish_reason_endoftext
  display_name: finish b/c endoftext
  description: Fraction of instances where the the output was terminated because the
    end of text token was generated.

- name: finish_reason_unknown
  display_name: finish b/c unknown
  description: Fraction of instances where the the output was terminated for unknown
    reasons.

- name: num_completions
  display_name: "# completions"
  description: Number of completions.

- name: predicted_index
  display_name: Predicted index
  description: Integer index of the reference (0, 1, ...) that was predicted by the
    model (for multiple-choice).

- name: training_co2_cost
  display_name: Estimated training emissions (kg CO2)
  short_display_name: Training emissions (kg CO2)
  lower_is_better: true
  description: Estimate of the CO2 emissions from training the model.

- name: training_energy_cost
  display_name: Estimated training energy cost (MWh)
  short_display_name: Training energy (MWh)
  lower_is_better: true
  description: Estimate of the amount of energy used to train the model.

- name: inference_runtime
  display_name: Observed inference runtime (s)
  short_display_name: Observed inference time (s)
  lower_is_better: true
  description: Average observed time to process a request to the model (via an API,
    and thus depends on particular deployment).

- name: inference_idealized_runtime
  display_name: Idealized inference runtime (s)
  short_display_name: Idealized inference time (s)
  lower_is_better: true
  description: Average time to process a request to the model based solely on the
    model architecture (using Megatron-LM).

- name: inference_denoised_runtime
  display_name: Denoised inference runtime (s)
  short_display_name: Denoised inference time (s)
  lower_is_better: true
  description: Average time to process a request to the model minus performance contention
    by using profiled runtimes from multiple trials of SyntheticEfficiencyScenario.

- name: batch_size
  display_name: Batch size
  description: For batch jobs, how many requests are in a batch.

- name: fid
  display_name: FID
  short_display_name: FID
  description: Fréchet Inception Distance (FID) is a metric used for evaluating the
    quality of images generated by models. The FID compares the distribution of generated
    images with the distribution of real images in feature space.
  lower_is_better: true

- name: inception_score
  display_name: Inception Score (IS)
  short_display_name: Inception Score
  description: Inception score (IS) is a metric used for evaluating the quality of
    image-based generative models. The score is calculated based on the output of
    a separate, pretrained Inception v3 image classification model applied to a sample
    of generated images.
  lower_is_better: false

- name: kernel_inception_distance
  display_name: Kernel Inception Distance (KID)
  short_display_name: Kernel Inception Distance
  description: Kernel Inception Distance (KID) is a metric used to assess the quality
    of image-based generative models. It was proposed to replace FID.
  lower_is_better: true

- name: expected_lpips_score
  display_name: Expected Learned Perceptual Image Patch Similarity (LPIPS) score
  short_display_name: Expected LPIPS score
  description: The Learned Perceptual Image Patch Similarity (LPIPS) is used to judge
    the perceptual similarity between two images. LPIPS computes the similarity between
    the activations of two image patches for some pre-defined network.
  lower_is_better: true

- name: expected_multi_scale_ssim_score
  display_name: Expected Multi-scale Structural Similarity Index Measure (SSIM)
  short_display_name: Expected Multi-Scale SSIM
  description: The Multi-scale Structural Similarity Index Measure (MS-SSIM) is measure
    of image quality and a generalization of Structural Similarity Index Measure (SSIM)
    by incorporating image details at different resolution scores.
  lower_is_better: false

- name: expected_psnr_score
  display_name: Expected Peak Signal-to-Noise Ratio (PSNR)
  short_display_name: Expected PSNR
  description: Peak signal-to-noise ratio (PSNR) is the ratio between the maximum
    possible power of a signal and the power of corrupting noise that affects the
    fidelity of its representation.
  lower_is_better: false

- name: expected_uiqi_score
  display_name: Expected Universal Image Quality Index (UIQI)
  short_display_name: Expected UIQI
  description: The Universal Image Quality Index (UIQI) is a full-reference image
    quality assessment method that measures the similarity between two images by comparing
    their luminance, contrast, and structure.
  lower_is_better: false

- name: gender_imbalance
  display_name: Gender imbalance
  short_display_name: Gender imbalance
  description: The gender imbalance metric assesses the presence of binary gender
    bias within a set of images.
  lower_is_better: true

- name: skin_tone_imbalance
  display_name: Skin tone imbalance
  short_display_name: Skin tone imbalance
  description: This skin tone metric assesses the presence of skin tone bias within
    a set of images.
  lower_is_better: true

- name: expected_clip_score
  display_name: Expected CLIP score
  short_display_name: Expected CLIP score
  description: CLIPscore measures how well an image is aligned with a corresponding
    natural language description, using the pre-trained CLIP model.
  lower_is_better: false

- name: max_clip_score
  display_name: Maximum CLIP score
  short_display_name: Max CLIP score
  description: CLIPscore measures how well an image is aligned with a corresponding
    natural language description, using the pre-trained CLIP model.
  lower_is_better: false

- name: detection_correct_frac
  display_name: Detection correct fraction
  short_display_name: Detection correct frac
  description: Fraction of correct images according to the ViTDet object detector
    with ViT-B backbone and detectron2 library to automatically detect objects specified
    in the prompts.
  lower_is_better: false

- name: expected_aesthetics_score
  display_name: Expected aesthetics score
  short_display_name: Expected aesthetics score
  description: Following LAION-Aesthetics for determining the aesthetic score of a
    generated image.
  lower_is_better: false

- name: max_aesthetics_score
  display_name: Maximum aesthetics score
  short_display_name: Max aesthetics score
  description: Following LAION-Aesthetics for determining the aesthetic score of a
    generated image.
  lower_is_better: false

- name: aesthetics_human
  display_name: Aesthetics (human)
  short_display_name: Aesthetics (human)
  description: How aesthetically pleasing is the image according to human evaluators.
  lower_is_better: false

- name: image_text_alignment_human
  display_name: Image text alignment (human)
  short_display_name: Image text alignment (human)
  description: How well does the image match the description according to human evaluators.
  lower_is_better: false

- name: clear_subject_human
  display_name: Clear subject (human)
  short_display_name: Clear subject (human)
  description: Is it clear who the subject(s) of the image is according to human evaluators.
  lower_is_better: false

- name: originality_human
  display_name: Originality (human)
  short_display_name: Originality (human)
  description: How original is the image, given it was created with the description
    according to human evaluators.
  lower_is_better: false

- name: copyright_human
  display_name: Copyright (human)
  short_display_name: Copyright (human)
  description: Whether the image is a derivative work of another image according to
    human evaluators.
  lower_is_better: true

- name: toxic_prompt_perspective_api
  display_name: Toxic prompt (Perspective API)
  short_display_name: Toxic prompt (Perspective API)
  description: Whether the prompt is toxic according to the Perspective API
  lower_is_better: true

- name: toxic_prompt_openai
  display_name: Toxic prompt (OpenAI)
  short_display_name: Toxic prompt (OpenAI)
  description: Whether the prompt violated [OpenAI's content policy](https://labs.openai.com/policies/content-policy).
  lower_is_better: true

- name: expected_max_nsfw_score
  display_name: Expected maximum NSFW score
  short_display_name: Expected max NSFW score
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: expected_max_nsfw_score_given_toxic_prompt=True
  display_name: Expected maximum NSFW score given toxic prompt
  short_display_name: Expected max NSFW score given toxic prompt
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: expected_max_nsfw_score_given_toxic_prompt=False
  display_name: Expected maximum NSFW score given non-toxic prompt
  short_display_name: Expected max NSFW score given non-toxic prompt
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: fractal_dimension_loss
  display_name: Fractal dimension loss
  short_display_name: Fractal dimension loss
  description: Fractal coefficient is a measure of the fractal patterns in an image.
    The ideal value is 1.4.
  lower_is_better: true

- name: max_nsfw_probability
  display_name: Expected maximum NSFW probability
  short_display_name: Expected max NSFW probability
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: max_nsfw_probability_given_toxic_prompt=True
  display_name: Expected maximum NSFW probability given toxic prompt
  short_display_name: Expected max NSFW probability given toxic prompt
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: max_nsfw_probability_given_toxic_prompt=False
  display_name: Expected maximum NSFW probability given non-toxic prompt
  short_display_name: Expected max NSFW probability given non-toxic prompt
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: nsfw_frac
  display_name: NSFW frac
  short_display_name: NSFW frac
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: nsfw_frac_given_toxic_prompt=True
  display_name: NSFW frac given toxic prompt
  short_display_name: NSFW frac given toxic prompt
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: nsfw_frac_given_toxic_prompt=False
  display_name: NSFW frac given non-toxic prompt
  short_display_name: NSFW frac given non-toxic prompt
  description: NSFW (not safe for work) classifier from LAION
  lower_is_better: true

- name: nudity_frac
  display_name: Nudity frac
  short_display_name: Nudity frac
  description: Pre-trained NudeNet to determine whether an image contains nudity.
  lower_is_better: true

- name: blacked_out_frac
  display_name: Blacked out frac
  short_display_name: Blacked out frac
  description: This metric represents whether the image has been blacked out.
  lower_is_better: true

- name: blacked_out_frac_given_toxic_prompt=True
  display_name: Blacked out frac given toxic prompt
  short_display_name: Blacked out frac given toxic prompt
  description: This metric represents whether the image has been blacked out.
  lower_is_better: true

- name: blacked_out_frac_given_toxic_prompt=False
  display_name: Blacked out frac given non-toxic prompt
  short_display_name: Blacked out frac given non-toxic prompt
  description: This metric represents whether the image has been blacked out.
  lower_is_better: true

- name: watermark_frac
  display_name: Watermark frac
  short_display_name: Watermark frac
  description: Watermark detector from LAION to determine whether an image contains
    watermarks.
  lower_is_better: true

- name: expected_max_watermark_prob
  display_name: Expected maximum watermark prob
  short_display_name: Expected max watermark prob
  description: Watermark detector from LAION to determine whether an image contains
    watermarks.
  lower_is_better: true

- name: photorealism_generated_human
  display_name: Photorealism w/ generated images (human)
  short_display_name: Photorealism - generated (human)
  description: Determine if the following image is AI-generated or real according
    to human raters.
  lower_is_better: false

- name: photorealism_real_human
  display_name: Photorealism w/ real images (human)
  short_display_name: Photorealism - real (human)
  description: Determine if the following image is AI-generated or real according
    to human raters.
  lower_is_better: false

- name: denoised_runtime
  display_name: Denoised runtime (in seconds)
  short_display_name: Denoised runtime (s)
  description: Denoised runtime is the runtime with the performance variance factored
    out as described [here](https://arxiv.org/abs/2305.02440).
  lower_is_better: true

- name: prompt_length
  display_name: Prompt length (number of characters)
  short_display_name: Prompt length (characters)
  description: The number of characters in the prompt
  lower_is_better: false

- name: inference_runtime
  display_name: Inference runtime (in seconds)
  short_display_name: Inference runtime (s)
  description: How long it took to generate the images
  lower_is_better: true

- name: num_generated_images
  display_name: Number of generated images
  short_display_name: Number of generated images
  description: The number of images the model generated
  lower_is_better: false
perturbations:

- name: robustness
  display_name: Robustness
  description: Computes worst case over different robustness perturbations (misspellings,
    formatting, contrast sets).

- name: fairness
  display_name: Fairness
  description: Computes worst case over different fairness perturbations (changing
    dialect, race of names, gender).

- name: typos
  display_name: Typos
  description: 'Randomly adds typos to each token in the input with probability 0.05
    and computes the per-instance worst-case performance between perturbed and unperturbed
    versions.'

- name: synonym
  display_name: Synonyms
  description: 'Randomly substitutes words in the input with WordNet synonyms with
    probability 0.5 and computes the per-instance worst-case performance between perturbed
    and unperturbed versions.'

- name: dialect
  display_name: SAE -> AAE
  short_display_name: Dialect
  description: 'Deterministically substitutes SAE words in input with AAE counterparts
    using validated dictionary of [Ziems et al. (2022)](https://aclanthology.org/2022.acl-long.258/)
    and computes the per-instance worst-case performance between perturbed and unperturbed
    versions.'

- name: race
  display_name: First names by race (White -> Black)
  short_display_name: Race
  description: 'Deterministically substitutes White first names with Black first names
    sampled from the lists of [Caliskan et al. (2017)](https://www.science.org/doi/10.1126/science.aal4230)
    and computes the per-instance worst-case performance between perturbed and unperturbed
    versions.'

- name: gender
  display_name: Pronouns by gender (Male -> Female)
  short_display_name: Gender
  description: 'Deterministically substitutes male pronouns with female pronouns and
    computes the per-instance worst-case performance between perturbed and unperturbed
    versions.'

- name: translate
  display_name: Translate
  short_display_name: Translate
  description: 'Translate text to other languages.'

############################################################
metric_groups:

- name: efficiency
  display_name: Efficiency
  metrics:
  - name: inference_denoised_runtime
    split: "${main_split}"

- name: efficiency_detailed
  display_name: Efficiency
  description: The efficiency of the model across both training and inference.
  metrics:
  - name: inference_runtime
    split: "${main_split}"
  - name: inference_idealized_runtime
    split: "${main_split}"
  - name: inference_denoised_runtime
    split: "${main_split}"
  - name: training_co2_cost
    split: "${main_split}"
  - name: training_energy_cost
    split: "${main_split}"

- name: general_information
  display_name: General information
  metrics:
  - name: num_instances
    split: "${main_split}"
  - name: prompt_truncated
    split: "${main_split}"
  - name: num_prompt_tokens
    split: "${main_split}"

- name: heim_inception
  display_name: HEIM inception metrics
  metrics:
  - name: fid
    split: __all__
  - name: inception_score
    split: __all__
  - name: kernel_inception_distance
    split: __all__

- name: heim_fidelity
  display_name: HEIM Image fidelity metrics
  metrics:
  - name: expected_lpips_score
    split: "${main_split}"
  - name: expected_multi_scale_ssim_score
    split: "${main_split}"
  - name: expected_psnr_score
    split: "${main_split}"
  - name: expected_uiqi_score
    split: "${main_split}"

- name: heim_gender
  display_name: HEIM gender metrics
  metrics:
  - name: gender_imbalance
    split: "${main_split}"

- name: heim_skin_tone
  display_name: HEIM skin tone metrics
  metrics:
  - name: skin_tone_imbalance
    split: "${main_split}"

- name: heim_alignment_human_metrics
  display_name: Image-text Alignment - Human Evaluation
  metrics:
  - name: image_text_alignment_human
    split: __all__
    perturbation_name: __all__

- name: heim_alignment_clip_metrics
  display_name: Image-text Alignment - CLIP Score
  metrics:
  - name: expected_clip_score
    split: "${main_split}"
  - name: max_clip_score
    split: "${main_split}"

- name: heim_quality_human_metrics
  display_name: Quality - Human Evaluation
  metrics:
  - name: photorealism_generated_human
    split: __all__
    perturbation_name: __all__

- name: heim_quality_fid_metrics
  display_name: Quality - FID
  metrics:
  - name: fid
    split: __all__

- name: heim_quality_inception_metrics
  display_name: Quality - Inception
  metrics:
  - name: inception_score
    split: __all__
  - name: kernel_inception_distance
    split: __all__

- name: heim_quality_automated_metrics
  display_name: Quality - Other Automated Metrics
  metrics:
  - name: expected_lpips_score
    split: "${main_split}"
  - name: expected_multi_scale_ssim_score
    split: "${main_split}"
  - name: expected_psnr_score
    split: "${main_split}"
  - name: expected_uiqi_score
    split: "${main_split}"

- name: heim_aesthetics_human_metrics
  display_name: Aesthetics - Human Evaluation
  metrics:
  - name: clear_subject_human
    split: __all__
    perturbation_name: __all__
  - name: aesthetics_human
    split: __all__
    perturbation_name: __all__

- name: heim_aesthetics_automated_metrics
  display_name: Aesthetics - Automated Metrics
  metrics:
  - name: expected_aesthetics_score
    split: "${main_split}"
  - name: max_aesthetics_score
    split: "${main_split}"
  - name: fractal_dimension_loss
    split: "${main_split}"

- name: heim_originality_human_metrics
  display_name: Originality - Human Evaluation
  metrics:
  - name: originality_human
    split: __all__
    perturbation_name: __all__

- name: heim_originality_watermark_metrics
  display_name: Originality - Watermark
  metrics:
  - name: watermark_frac
    split: "${main_split}"

- name: heim_reasoning_metrics
  display_name: Reasoning
  metrics:
  - name: detection_correct_frac
    split: "${main_split}"

- name: heim_bias_metrics
  display_name: Bias
  metrics:
  - name: gender_imbalance
    split: "${main_split}"
  - name: skin_tone_imbalance
    split: "${main_split}"

- name: heim_gender_metrics
  display_name: Gender imbalance
  metrics:
  - name: gender_imbalance
    split: "${main_split}"

- name: heim_skin_tone_metrics
  display_name: Skin tone imbalance
  metrics:
  - name: skin_tone_imbalance
    split: "${main_split}"

- name: heim_toxicity_metrics
  display_name: Toxicity (NSFW)
  metrics:
  - name: nsfw_frac_given_toxic_prompt=False
    split: "${main_split}"

- name: heim_toxicity_black_metrics
  display_name: Toxicity (blacked out images)
  metrics:
  - name: blacked_out_frac
    split: "${main_split}"

- name: heim_toxicity_nudity_metrics
  display_name: Toxicity (nudity)
  metrics:
  - name: nudity_frac
    split: "${main_split}"

- name: heim_efficiency_metrics
  display_name: Efficiency
  metrics:
  - name: denoised_runtime
    split: __all__

- name: heim_photorealism
  display_name: HEIM photorealism metrics
  metrics:
  - name: photorealism_generated_human
    split: __all__
    perturbation_name: __all__

############################################################
run_groups:

- name: core_scenarios
  display_name: All scenarios
  description: All scenarios
  category: All scenarios
  subgroups:
  - mscoco_base
  - mscoco_fid
  - mscoco_efficiency
  - mscoco_gender
  - mscoco_dialect
  - mscoco_robustness
  - mscoco_chinese
  - mscoco_hindi
  - mscoco_spanish
  - mscoco_art_styles
  - cub200
  - draw_bench_image_quality
  - parti_prompts_image_quality
  - daily_dalle
  - landing_page
  - logos
  - magazine_cover
  - common_syntactic_processes
  - draw_bench_reasoning
  - parti_prompts_reasoning
  - relational_understanding
  - detection
  - winoground
  - parti_prompts_knowledge
  - draw_bench_knowledge
  - time_most_significant_historical_figures
  - demographic_stereotypes
  - mental_disorders
  - i2p

- name: heim_alignment_scenarios
  display_name: Alignment
  description: Is the image semantically correct given the text (image-text alignment)
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_base
  - cub200
  - draw_bench_image_quality
  - draw_bench_reasoning
  - draw_bench_knowledge
  - parti_prompts_image_quality
  - parti_prompts_reasoning
  - parti_prompts_knowledge

- name: heim_quality_scenarios
  display_name: Quality
  description: Do the generated images look like real images/photos
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_base
  - draw_bench_image_quality
  - parti_prompts_image_quality

- name: heim_aesthetics_scenarios
  display_name: Aesthetics
  description: Is the image aesthetically pleasing
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_base
  - mscoco_art_styles
  - daily_dalle
  - logos
  - landing_page
  - magazine_cover

- name: heim_originality_scenarios
  display_name: Originality
  description: Does the model generate creative images and prevent copyright infringement
  category: Scenarios for specific aspects
  subgroups:
  - daily_dalle
  - logos
  - landing_page
  - magazine_cover

- name: heim_reasoning_scenarios
  display_name: Reasoning
  description: Does the model understand objects, counts, and spatial relations (compositionality)
  category: Scenarios for specific aspects
  subgroups:
  - common_syntactic_processes
  - draw_bench_reasoning
  - parti_prompts_reasoning
  - relational_understanding
  - detection
  - winoground

- name: heim_knowledge_scenarios
  display_name: Knowledge
  description: Does the model have knowledge about the world or domains
  category: Scenarios for specific aspects
  subgroups:
  - time_most_significant_historical_figures
  - draw_bench_knowledge
  - parti_prompts_knowledge

- name: heim_bias_scenarios
  display_name: Bias
  description: Are the generated images biased in demographic representation (e.g.,
    gender, skin tone)
  category: Scenarios for specific aspects
  subgroups:
  - demographic_stereotypes
  - mental_disorders

- name: heim_toxicity_scenarios
  display_name: Toxicity
  description: Does the model generate toxic or inappropriate images (e.g., violence,
    sexual, illegal content)
  category: Scenarios for specific aspects
  subgroups:
  - i2p

- name: heim_fairness_dialect_scenarios
  display_name: Fairness - African American Vernacular English (AAVE)
  description: Does the model exhibit performance disparities across social groups.
    The African American Vernacular English (AAVE) dialect perturbation converts each
    word to the corresponding word in AAVE if one exists.
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_dialect

- name: heim_fairness_gender_scenarios
  display_name: Fairness - Gender
  description: Does the model exhibit performance disparities across social groups.
    The gender perturbation maps male gender terms to female gender terms (e.g., son
    to daughter and father to mother).
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_gender

- name: heim_robustness_scenarios
  display_name: Robustness
  description: Is the model robust to invariant input perturbations
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_robustness

- name: heim_multilinguality_chinese_scenarios
  display_name: Multilinguality (Chinese)
  description: Does the model support non-English languages.
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_chinese

- name: heim_multilinguality_hindi_scenarios
  display_name: Multilinguality (Hindi)
  description: Does the model support non-English languages.
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_hindi

- name: heim_multilinguality_spanish_scenarios
  display_name: Multilinguality (Spanish)
  description: Does the model support non-English languages.
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_spanish

- name: heim_fid_scenarios
  display_name: Fidelity
  description: Fidelity metrics computed with MS-COCO.
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_fid

- name: heim_efficiency_scenarios
  display_name: Efficiency
  description: How fast is inference for the model
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_efficiency

- name: heim_art_styles_scenarios
  display_name: Art styles
  description: To test the ability of these models to generate images in specific
    art styles.
  category: Scenarios for specific aspects
  subgroups:
  - mscoco_art_styles

- name: mscoco
  display_name: MS-COCO (all)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_quality_human_metrics
  - heim_quality_automated_metrics
  - heim_aesthetics_human_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_base
  display_name: MS-COCO (base)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_quality_human_metrics
  - heim_quality_automated_metrics
  - heim_aesthetics_human_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_fid
  display_name: MS-COCO Fidelity
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    for fidelity. To compute the FID, we randomly selected 30,000 text prompts from
    MS-COCO and generated a single image for each prompt using the text-to-image generation
    model that we are evaluating. Then, we used [pytorch-fid](https://github.com/mseitzer/pytorch-fid)
    to compute the FID between the set of real images associated with the prompts
    and the set of generated images.
  metric_groups:
  - heim_quality_fid_metrics
  - heim_quality_inception_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_efficiency
  display_name: MS-COCO Efficiency
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    for efficiency.
  metric_groups:
  - heim_efficiency_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Efficiency

- name: mscoco_gender
  display_name: MS-COCO (fairness - gender)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with the gender perturbation maps male gender terms to female gender terms (e.g.,
    son to daughter and father to mother).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_dialect
  display_name: MS-COCO (fairness - AAVE dialect)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with the African American Vernacular English (AAVE) dialect perturbation, which
    converts each word to the corresponding word in AAVE if one exists.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_robustness
  display_name: MS-COCO (robustness)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with input perturbations.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_chinese
  display_name: MS-COCO (Chinese)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with prompts translated to Chinese.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_hindi
  display_name: MS-COCO (Hindi)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with prompts translated to Hindi.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_spanish
  display_name: MS-COCO (Spanish)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with prompts translated to Spanish.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: mscoco_art_styles
  display_name: MS-COCO (Art styles)
  description: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))
    with prompts that generate images in specific art styles.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_quality_human_metrics
  - heim_aesthetics_human_metrics
  - general_information
  environment:
    main_split: valid
  taxonomy:
    task: Image quality

- name: cub200
  display_name: Caltech-UCSD Birds-200-2011
  description: Caltech-UCSD Birds-200-2011 is a challenging dataset of 200 bird species
    with 10 captions for each bird ([paper](https://authors.library.caltech.edu/27452/1/CUB_200_2011.pdf),
    [paper](https://arxiv.org/abs/1711.10485)).
  metric_groups:
  - heim_alignment_clip_metrics
  - heim_quality_automated_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Image quality

- name: draw_bench_image_quality
  display_name: DrawBench (image quality categories)
  description: A comprehensive and challenging set of prompts that support the evaluation
    and comparison of text-to-image models ([paper](https://arxiv.org/abs/2205.11487)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Image quality

- name: parti_prompts_image_quality
  display_name: PartiPrompts (image quality categories)
  description: PartiPrompts (P2) is a set of 1600 diverse English prompts that allow
    us to more comprehensively evaluate and test the limits of text-to-image synthesis
    models. ([paper](https://arxiv.org/abs/2206.10789))
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Image quality

- name: daily_dalle
  display_name: dailydall.e
  description: DALL-E 2 prompts from [Chad Nelson's Instagram](https://www.instagram.com/dailydall.e/)
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_human_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_human_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Originality

- name: landing_page
  display_name: Landing Page
  description: Prompts to generate landing pages for mobile or web applications.
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_human_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_human_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Originality

- name: logos
  display_name: Logos
  description: Prompts to generate logos for brands and companies
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_human_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_human_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Originality

- name: magazine_cover
  display_name: Magazine Cover Photos
  description: Prompts to generate magazine cover photos
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_human_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_human_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Originality

- name: common_syntactic_processes
  display_name: Common Syntactic Processes
  description: Prompts from 8 different grammatical phenomena ([paper](https://arxiv.org/abs/2210.12889)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Reasoning

- name: draw_bench_reasoning
  display_name: DrawBench (reasoning categories)
  description: A comprehensive and challenging set of prompts that support the evaluation
    and comparison of text-to-image models ([paper](https://arxiv.org/abs/2205.11487)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Reasoning

- name: parti_prompts_reasoning
  display_name: PartiPrompts (reasoning categories)
  description: PartiPrompts (P2) is a set of 1600 diverse English prompts that allow
    us to more comprehensively evaluate and test the limits of text-to-image synthesis
    models. ([paper](https://arxiv.org/abs/2206.10789))
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Reasoning

- name: relational_understanding
  display_name: Relational Understanding
  description: Based on existing cognitive, linguistic, and developmental literature,
    the authors created a set of 15 relations (8 physical, 7 agentic) and a set of
    12 entities (6 objects, 6 agents) ([paper](https://arxiv.org/abs/2208.00005))
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Reasoning

- name: detection
  display_name: Detection (PaintSkills)
  description: A compositional diagnostic dataset an evaluation toolkit that measures
    visual reasoning skills ([paper](https://arxiv.org/abs/2202.04053)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_reasoning_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Reasoning

- name: winoground
  display_name: Winoground
  description: Winoground is a novel task and dataset for evaluating the ability of
    vision and language models to conduct visio-linguistic compositional reasoning
    ([paper](https://arxiv.org/abs/2204.03162)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_quality_automated_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Reasoning

- name: parti_prompts_knowledge
  display_name: PartiPrompts (knowledge categories)
  description: PartiPrompts (P2) is a set of 1600 diverse English prompts that allow
    us to more comprehensively evaluate and test the limits of text-to-image synthesis
    models. ([paper](https://arxiv.org/abs/2206.10789))
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Knowledge

- name: draw_bench_knowledge
  display_name: DrawBench (knowledge categories)
  description: A comprehensive and challenging set of prompts that support the evaluation
    and comparison of text-to-image models ([paper](https://arxiv.org/abs/2205.11487)).
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Knowledge

- name: time_most_significant_historical_figures
  display_name: TIME's most significant historical figures
  description: People from TIME's "The 100 Most Significant Figures in History" list
  metric_groups:
  - heim_alignment_human_metrics
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Knowledge

- name: demographic_stereotypes
  display_name: Demographic Stereotypes
  description: Simple user prompts ("A photo of the face of...") generate images perpetuating
    dangerous racial, ethnic, gendered, class, and intersectional stereotypes.
  metric_groups:
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_bias_metrics
  - heim_gender_metrics
  - heim_skin_tone_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Unbiasedness

- name: mental_disorders
  display_name: Mental Disorders
  description: Prompt text-to-image models with mental disorders listed by the World
    Health Organization
  metric_groups:
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_bias_metrics
  - heim_gender_metrics
  - heim_skin_tone_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Unbiasedness

- name: i2p
  display_name: Inappropriate Image Prompts (I2P)
  description: Contains real user prompts for generative text-to-image prompts that
    are likely to produce inappropriate images ([paper](https://arxiv.org/abs/2211.05105)).
  metric_groups:
  - heim_alignment_clip_metrics
  - heim_aesthetics_automated_metrics
  - heim_originality_watermark_metrics
  - heim_toxicity_metrics
  - heim_toxicity_black_metrics
  - heim_toxicity_nudity_metrics
  - general_information
  environment:
    main_split: test
  taxonomy:
    task: Toxicity mitigation
