---
############################################################
# For backwards compatibility with older versions of HELM.
# TODO: Remove this after 2024-09-01.
adapter: []
############################################################
metrics:
  # Infrastructure metrics:
  - name: num_references
    display_name: '# ref'
    description: Number of references.
  - name: num_train_trials
    display_name: '# trials'
    description: Number of trials, where in each trial we choose an independent, random set of training instances.
  - name: estimated_num_tokens_cost
    display_name: 'cost'
    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
  - name: num_prompt_tokens
    display_name: '# prompt tokens'
    description: Number of tokens in the prompt.
  - name: num_prompt_characters
    display_name: '# prompt chars'
    description: Number of characters in the prompt.
  - name: num_completion_tokens
    display_name: '# completion tokens'
    description: Actual number of completion tokens (over all completions).
  - name: num_output_tokens
    display_name: '# output tokens'
    description: Actual number of output tokens.
  - name: max_num_output_tokens
    display_name: 'Max output tokens'
    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
  - name: num_requests
    display_name: '# requests'
    description: Number of distinct API requests.
  - name: num_instances
    display_name: '# eval'
    description: Number of evaluation instances.
  - name: num_train_instances
    display_name: '# train'
    description: Number of training instances (e.g., in-context examples).
  - name: prompt_truncated
    display_name: truncated
    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
  - name: finish_reason_length
    display_name: finish b/c length
    description: Fraction of instances where the the output was terminated because of the max tokens limit.
  - name: finish_reason_stop
    display_name: finish b/c stop
    description: Fraction of instances where the the output was terminated because of the stop sequences.
  - name: finish_reason_endoftext
    display_name: finish b/c endoftext
    description: Fraction of instances where the the output was terminated because the end of text token was generated.
  - name: finish_reason_unknown
    display_name: finish b/c unknown
    description: Fraction of instances where the the output was terminated for unknown reasons.
  - name: num_completions
    display_name: '# completions'
    description: Number of completions.
  - name: predicted_index
    display_name: Predicted index
    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).

  # Vision Language metrics [text]:
  - name: edit_similarity
    display_name: Edit similarity (Levenshtein)
    short_display_name: Edit sim.
    lower_is_better: false
    description: Average Levenshtein edit similarity (1 - distance normalized by length of longer sequence) between model generation and reference.

  # Vision Language metrics [image]:
  - name: earth_mover_similarity
    display_name: Earth Mover Similarity
    short_display_name: EMS
    description: Earth Mover Similarity (EMD adapted to speed up calculations)
    lower_is_better: false
  - name: pixel_similarity
    display_name: Pixel Similarity
    short_display_name: PS
    description: Pixel Similarity between an image generated by the model and the target image.
    lower_is_better: false
  - name: compilation_success
    display_name: Compilation success
    description: Fraction of instances where the generated code compiles successfully.
    lower_is_better: false
  - name: fid_similarity
    display_name: CIS
    short_display_name: CIS
    description: The cosine similarity between the Inception feature vectors.
    lower_is_better: false
  - name: lpips_similarity
    display_name: LPIPS
    short_display_name: LPIPS
    description: The LPIPS distance between the generated image and the target image.
    lower_is_better: false
  - name: ssim_similarity
    display_name: SSIM
    short_display_name: SSIM
    description: The SSIM similarity between the generated image and the target image.
    lower_is_better: false

  # Accuracy metrics:
  - name: exact_match
    display_name: Exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference exactly.
    lower_is_better: false
  - name: quasi_exact_match
    display_name: Quasi-exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
    lower_is_better: false
  - name: prefix_exact_match
    display_name: Prefix exact match
    short_display_name: PEM
    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
    lower_is_better: false
  - name: quasi_prefix_exact_match
    # TODO: should call this prefix_quasi_exact_match
    display_name: Prefix quasi-exact match
    short_display_name: PEM
    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
    lower_is_better: false

############################################################
perturbations:
  - name: robustness
    display_name: Robustness
    description: Computes worst case over different robustness perturbations (misspellings, formatting, contrast sets).

############################################################
metric_groups:
  - name: accuracy
    display_name: Compilation Rate and Earth Mover Similarity
    metrics:
      - name: earth_mover_similarity
        split: ${main_split}
      - name: compilation_success
        split: ${main_split}

  - name: general_information
    display_name: General information
    metrics:
      - name: num_instances
        split: ${main_split}
      - name: num_train_instances
        split: ${main_split}
      - name: prompt_truncated
        split: ${main_split}
      - name: num_prompt_tokens
        split: ${main_split}
      - name: num_output_tokens
        split: ${main_split}

  - name: accuracy_simple
    display_name: Earth Mover Similarity
    metrics:
      - name: earth_mover_similarity
        split: ${main_split}

  - name: compilation
    display_name: Compilation Rate
    metrics:
      - name: compilation_success
        split: ${main_split}

  - name: generation_image
    display_name: Generation (image)
    metrics:
      - name: pixel_similarity
        split: ${main_split}
      - name: compilation_success
        split: ${main_split}
      - name: fid_similarity
        split: ${main_split}
      - name: earth_mover_similarity
        split: ${main_split}
      - name: lpips_similarity
        split: ${main_split}
      - name: ssim_similarity
        split: ${main_split}

  - name: generation_text
    display_name: Generation (text)
    metrics:
      - name: edit_similarity
        split: ${main_split}

############################################################
run_groups:
  - name: core_scenarios
    display_name: Image2Struct
    description: Scenarios for evaluating the ability of Vision-Language models to generate structured outputs from images.
    category: All scenarios
    subgroups:
      - image2latex
      - image2webpage
      - image2musicsheet

  - name: image2struct_wild
    display_name: Image2Struct (Wild)
    description: Scenarios for evaluating the ability of Vision-Language models to generate structured outputs from images. These scenarios contain images that do not have a ground truth.
    category: All scenarios
    subgroups:
      - image2latex_wild
      - image2webpage_wild

  - name: image2latex
    display_name: Image2LaTeX
    description: The Image2LaTeX benchmark for converting images of mathematical equations, tables. algorithms and tikz to LaTeX.
    metric_groups:
      - accuracy
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: mathematical equations, tables, algorithms, tikz
      who: dataset authors
      when: "2024"
      language: English

  - name: image2latex_equation
    display_name: I2LaTeX (Equation)
    description: The Image2LaTeX benchmark subset for converting images of mathematical equations to LaTeX.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: mathematical equations
      who: dataset authors
      when: "2024"
      language: English

  - name: image2latex_table
    display_name: I2LaTeX (Table)
    description: The Image2LaTeX benchmark subset for converting images of tables to LaTeX.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: tables
      who: dataset authors
      when: "2024"

  - name: image2latex_algorithm
    display_name: I2LaTeX (Algorithm)
    description: The Image2LaTeX benchmark subset for converting images of algorithms to LaTeX.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: algorithms
      who: dataset authors
      when: "2024"

  - name: image2latex_plot
    display_name: I2LaTeX (Tikz)
    description: The Image2LaTeX benchmark subset for converting images of tikz to LaTeX.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: tikz (plots)
      who: dataset authors
      when: "2024"

  # - name: image2latex_easy
  #   display_name: I2LaTeX (Easy)
  #   description: The 1/3 easiest examples of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - generation_text
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: mathematical equations, tables, algorithms, tikz
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  # - name: image2latex_medium
  #   display_name: I2LaTeX (Medium)
  #   description: The 1/3 examples with medium diffulty of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - generation_text
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: mathematical equations, tables, algorithms, tikz
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  # - name: image2latex_hard
  #   display_name: I2LaTeX (Hard)
  #   description: The 1/3 hardest examples of the Image2LaTeX benchmark according to a simple heuristic counting the number of characters in the ground truth LaTeX code.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - generation_text
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: mathematical equations, tables, algorithms, tikz
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  - name: image2latex_wild
    display_name: Image2LaTeX (Wild)
    description: Images of mathematical equations gathered from Wikipedia that do not have a LaTeX ground truth.
    metric_groups:
      - accuracy
      - generation_image
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: mathematical equations, tables, algorithms, tikz
      who: dataset authors
      when: "2024"
      language: English

  - name: image2webpage
    display_name: Image2webpage
    description: The Image2webpage benchmark for converting images of webpages to HTML/CSS/Javascript.
    metric_groups:
      - accuracy
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: css, html, javascript
      who: dataset authors
      when: "2024"
      language: English

  - name: image2webpage_css
    display_name: I2webpage (CSS)
    description: The Image2webpage benchmark subset for converting images of webpages to code repo containing mostly CSS.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: code (mostly CSS)
      who: dataset authors
      when: "2024"

  - name: image2webpage_html
    display_name: I2webpage (HTML)
    description: The Image2webpage benchmark subset for converting images of webpages to code repo containing mostly HTML.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: code (mostly HTML)
      who: dataset authors
      when: "2024"

  - name: image2webpage_javascript
    display_name: I2webpage (Javascript)
    description: The Image2webpage benchmark subset for converting images of webpages to code repo containing mostly Javascript.
    metric_groups:
      - accuracy_simple
      - compilation
      - generation_image
      - generation_text
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: code (mostly Javascript)
      who: dataset authors
      when: "2024"

  # - name: image2webpage_easy
  #   display_name: I2webpage (Easy)
  #   description: The 1/3 easiest examples of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - generation_text
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: css, html, javascript
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  # - name: image2webpage_medium
  #   display_name: I2webpage (Medium)
  #   description: The 1/3 examples with medium diffulty of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - generation_text
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: css, html, javascript
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  # - name: image2webpage_hard
  #   display_name: I2webpage (Hard)
  #   description: The 1/3 hardest examples of the Image2webpage benchmark according to a simple heuristic counting the number of characters in the ground truth HTML/CSS/Javascript code.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - generation_text
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: css, html, javascript
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  - name: image2webpage_wild
    display_name: Image2webpage (Wild)
    description: Images of webpages gathered from the internet by taking sceenshots and so on that do not have a HTML/CSS/Javascript ground truth.
    metric_groups:
      - accuracy
      - generation_image
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: css, html, javascript
      who: dataset authors
      when: "2024"
      language: English

  - name: image2musicsheet
    display_name: Image2musicsheet
    description: The Image2musicsheet benchmark for converting images of music sheets to LilyPond.
    metric_groups:
      - accuracy
      - generation_image
      - general_information
    environment:
      main_name: earth_mover_similarity
      main_split: valid
    taxonomy:
      task: image-to-text
      what: music sheets
      who: dataset authors
      when: "2024"
      language: English

  # - name: image2musicsheet_easy
  #   display_name: I2musicsheet (Easy)
  #   description: The 1/3 easiest examples of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: music sheets
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  # - name: image2musicsheet_medium
  #   display_name: I2musicsheet (Medium)
  #   description: The 1/3 examples with medium diffulty of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: music sheets
  #     who: dataset authors
  #     when: "2024"
  #     language: English

  # - name: image2musicsheet_hard
  #   display_name: I2musicsheet (Hard)
  #   description: The 1/3 hardest examples of the Image2musicsheet benchmark according to a simple heuristic counting the number of black pixels in the target image.
  #   metric_groups:
  #     - accuracy_simple
  #     - compilation
  #     - generation_image
  #     - general_information
  #   environment:
  #     main_name: earth_mover_similarity
  #     main_split: valid
  #   taxonomy:
  #     task: image-to-text
  #     what: music sheets
  #     who: dataset authors
  #     when: "2024"
  #     language: English