---

############################################################
metrics:
  # Infrastructure metrics:
  - name: num_perplexity_tokens
    display_name: '# tokens'
    description: Average number of tokens in the predicted output (for language modeling, the input too).
  - name: num_bytes
    display_name: '# bytes'
    description: Average number of bytes in the predicted output (for language modeling, the input too).
  - name: num_references
    display_name: '# ref'
    description: Number of references.
  - name: num_train_trials
    display_name: '# trials'
    description: Number of trials, where in each trial we choose an independent, random set of training instances.
  - name: estimated_num_tokens_cost
    display_name: 'cost'
    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
  - name: num_prompt_tokens
    display_name: '# prompt tokens'
    description: Number of tokens in the prompt.
  - name: num_prompt_characters
    display_name: '# prompt chars'
    description: Number of characters in the prompt.
  - name: num_completion_tokens
    display_name: '# completion tokens'
    description: Actual number of completion tokens (over all completions).
  - name: num_output_tokens
    display_name: '# output tokens'
    description: Actual number of output tokens.
  - name: max_num_output_tokens
    display_name: 'Max output tokens'
    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
  - name: num_requests
    display_name: '# requests'
    description: Number of distinct API requests.
  - name: num_instances
    display_name: '# eval'
    description: Number of evaluation instances.
  - name: num_train_instances
    display_name: '# train'
    description: Number of training instances (e.g., in-context examples).
  - name: prompt_truncated
    display_name: truncated
    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
  - name: finish_reason_length
    display_name: finish b/c length
    description: Fraction of instances where the the output was terminated because of the max tokens limit.
  - name: finish_reason_stop
    display_name: finish b/c stop
    description: Fraction of instances where the the output was terminated because of the stop sequences.
  - name: finish_reason_endoftext
    display_name: finish b/c endoftext
    description: Fraction of instances where the the output was terminated because the end of text token was generated.
  - name: finish_reason_unknown
    display_name: finish b/c unknown
    description: Fraction of instances where the the output was terminated for unknown reasons.
  - name: num_completions
    display_name: '# completions'
    description: Number of completions.
  - name: predicted_index
    display_name: Predicted index
    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).

  # Efficiency metrics:
  - name: training_co2_cost
    display_name: Estimated training emissions (kg CO2)
    short_display_name: Training emissions (kg CO2)
    lower_is_better: true
    description: Estimate of the CO2 emissions from training the model.
  - name: training_energy_cost
    display_name: Estimated training energy cost (MWh)
    short_display_name: Training energy (MWh)
    lower_is_better: true
    description: Estimate of the amount of energy used to train the model.
  - name: inference_runtime
    display_name: Observed inference runtime (s)
    short_display_name: Observed inference time (s)
    lower_is_better: true
    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
  - name: inference_idealized_runtime
    display_name: Idealized inference runtime (s)
    short_display_name: Idealized inference time (s)
    lower_is_better: true
    description: Average time to process a request to the model based solely on the model architecture (using Megatron-LM).
  - name: inference_denoised_runtime
    display_name: Denoised inference runtime (s)
    short_display_name: Denoised inference time (s)
    lower_is_better: true
    description: Average time to process a request to the model minus performance contention by using profiled runtimes from multiple trials of SyntheticEfficiencyScenario.
  - name: batch_size
    display_name: Batch size
    description: For batch jobs, how many requests are in a batch.

  # Unitxt Metrics
  - name: meteor
    display_name: METEOR
    short_display_name: METEOR
    description: METEOR
    lower_is_better: false
  - name: f1
    display_name: BERTScore F1
    short_display_name: BERTScore F1
    description: BERTScore F1
    lower_is_better: false
  - name: precision
    display_name: Precision
    short_display_name: Precision
    description: Precision
    lower_is_better: false
  - name: recall
    display_name: Recall
    short_display_name: Recall
    description: Recall
    lower_is_better: false
  - name: rouge1
    display_name: ROUGE-1
    short_display_name: ROUGE-1
    description: ROUGE-1
    lower_is_better: false
  - name: rouge2
    display_name: ROUGE-2
    short_display_name: ROUGE-2
    description: ROUGE-2
    lower_is_better: false
  - name: rougeL
    display_name: ROUGE-L
    short_display_name: ROUGE-L
    description: ROUGE-L
    lower_is_better: false
  - name: rougeLsum
    display_name: ROUGE-Lsum
    short_display_name: ROUGE-Lsum
    description: ROUGE-Lsum
    lower_is_better: false
  - name: bleu
    display_name: BLEU
    short_display_name: BLEU
    description: BLEU
    lower_is_better: false
  - name: accuracy
    display_name: Accuracy
    short_display_name: Accuracy
    description: Accuracy
    lower_is_better: false
  - name: f1_macro
    display_name: Macro F1
    short_display_name: Macro F1
    description: Macro F1
    lower_is_better: false
  - name: f1_micro
    display_name: Micro F1
    short_display_name: Micro F1
    description: Micro F1
    lower_is_better: false
  - name: unsorted_list_exact_match
    display_name: Unsorted List Exact Match
    short_display_name: Exact Match
    description: Unsorted List Exact Match
    lower_is_better: false
  - name: f1_strings
    display_name: F1 Strings
    short_display_name: F1 Strings
    description: F1 Strings
    lower_is_better: false

  # FinQA Accuracy
  - name: program_accuracy
    display_name: Program Accuracy
    short_display_name: Program Accuracy
    description: Program Accuracy
    lower_is_better: false
  - name: execution_accuracy
    display_name: Execution Accuracy
    short_display_name: Execution Accuracy
    description: Execution Accuracy
    lower_is_better: false

  # SciGen Accuracy
  - name: llama_3_1_70b_instruct_cross_provider_template_table2text_single_turn_with_reference
    display_name: Rating
    short_display_name: Rating
    description: Rating by Llama 3.1 (70B) LLM as judge
    lower_is_better: false

  # Robustness
  # NOTE: This is a "virtual" metric that is not produced directly by the metrics, but will appear as an aggregate table.
  # Run `helm-summarize` with `--summarizer-class helm.benchmark.presentation.torr_robustness_summarizer.ToRRRobustnessSummarizer`
  # to compute the values of this metric.
  - name: robustness
    display_name: Robustness
    short_display_name: Robustness
    description: Robustness
    lower_is_better: false

perturbations: []

metric_groups:
  - name: performance_metrics
    display_name: Performance
    aggregation_strategies: 
      - mean
    metrics:
    - name: ${main_name}
      split: __all__

  # NOTE: Robustness is a "virtual" metric that is not produced directly by the metrics, but will appear as an aggregate table.
  # Run `helm-summarize` with `--summarizer-class helm.benchmark.presentation.torr_robustness_summarizer.ToRRRobustnessSummarizer`
  # to compute the values of this metric.
  - name: robustness_metrics
    display_name: Robustness
    aggregation_strategies: 
      - mean
    metrics:
    - name: robustness
      split: __all__

  - name: generation_metrics
    display_name: Other Generation Metrics
    hide_win_rates: true
    metrics:
    - name: f1
      split: __all__
    - name: rouge1
      split: __all__
    - name: rouge2
      split: __all__
    - name: rougeL
      split: __all__
    - name: rougeLsum
      split: __all__
    - name: bleu
      split: __all__

  - name: classification_metrics
    display_name: Classification Metrics
    hide_win_rates: true
    metrics:
    - name: accuracy
      split: __all__
    - name: f1_macro
      split: __all__
    - name: f1_micro
      split: __all__

  - name: efficiency
    display_name: Efficiency
    metrics:
    - name: inference_runtime
      split: ${main_split}

  - name: general_information
    display_name: General information
    hide_win_rates: true
    metrics:
    - name: num_instances
      split: ${main_split}
    - name: num_train_instances
      split: ${main_split}
    - name: prompt_truncated
      split: ${main_split}
    - name: num_prompt_tokens
      split: ${main_split}
    - name: num_output_tokens
      split: ${main_split}

run_groups:
  - name: table_scenarios
    display_name: Table Scenarios
    description: Table Scenarios
    category: All Scenarios
    subgroups:
      - fin_qa
      - numeric_nlg
      - qtsumm
      - scigen
      - tab_fact
      - tablebench_data_analysis
      - tablebench_fact_checking
      - tablebench_numerical_reasoning
      - turl_col_type
      - wikitq

  - name: fin_qa
    display_name: FinQA
    description: The FinQA benchmark for numeric reasoning over financial data, with question answering pairs written by financial experts over financial reports [(Chen et al., 2021)](https://arxiv.org/abs/2109.00122/).
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: program_accuracy
      main_split: test
    taxonomy:
      task: question answering with numeric reasoning
      what: financial reports
      who: financial experts
      when: 1999 to 2019
      language: English

  - name: numeric_nlg
    display_name: NumericNLG
    short_display_name: NumericNLG
    description: "NumericNLG is a dataset for numerical table-to-text generation using pairs of a table and a paragraph of a table description with richer inference from scientific papers."
    metric_groups:
      - performance_metrics
      - robustness_metrics
      # - generation_metrics
      - efficiency
      - general_information
    environment:
      main_name: rougeL
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: qtsumm
    display_name: QTSumm
    short_display_name: QTSumm
    description: QTFumm
    metric_groups:
      - performance_metrics
      - robustness_metrics
      # - generation_metrics
      - efficiency
      - general_information
    environment:
      main_name: rougeL
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: scigen
    display_name: SciGen
    description: SciGen
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: rougeL
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: tab_fact
    display_name: TabFact
    short_display_name: TabFact
    description: "tab_fact is a large-scale dataset for the task of fact-checking on tables."
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: accuracy
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: tablebench_data_analysis
    display_name: Tablebench Data Analysis
    short_display_name: Tablebench Data Analysis
    description: "tab_fact is a large-scale dataset for the task of fact-checking on tables."
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: rougeL
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: tablebench_fact_checking
    display_name: Tablebench Fact Checking
    short_display_name: Tablebench Fact Checking
    description: "tab_fact is a large-scale dataset for the task of fact-checking on tables."
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: rougeL
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: tablebench_numerical_reasoning
    display_name: Tablebench Numerical Reasoning
    short_display_name: Tablebench Numerical Reasoning
    description: "tab_fact is a large-scale dataset for the task of fact-checking on tables."
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: rougeL
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: turl_col_type
    display_name: Turl Col Type
    description: Turl Col Type
    metric_groups:
      - performance_metrics
      - robustness_metrics
      - efficiency
      - general_information
    environment:
      main_name: f1_micro
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English

  - name: wikitq
    display_name: WikiTableQuestions
    short_display_name: WikiTableQuestions
    description: "This WikiTableQuestions dataset is a large-scale dataset for the task of question answering on semi-structured tables."
    metric_groups:
      - performance_metrics
      - robustness_metrics
      # - classification_metrics
      - efficiency
      - general_information
    environment:
      main_name: f1_strings
      main_split: test
    taxonomy:
      task: "?"
      what: "?"
      who: "?"
      when: "?"
      language: English
