---
############################################################
metrics:
  # Infrastructure metrics:
  - name: num_perplexity_tokens
    display_name: '# tokens'
    description: Average number of tokens in the predicted output (for language modeling, the input too).
  - name: num_bytes
    display_name: '# bytes'
    description: Average number of bytes in the predicted output (for language modeling, the input too).

  - name: num_references
    display_name: '# ref'
    description: Number of references.
  - name: num_train_trials
    display_name: '# trials'
    description: Number of trials, where in each trial we choose an independent, random set of training instances.
  - name: num_prompt_tokens
    display_name: '# prompt tokens'
    description: Number of tokens in the prompt.
  - name: num_completion_tokens
    display_name: '# completion tokens'
    description: Actual number of completion tokens (over all completions).
  - name: num_output_tokens
    display_name: '# output tokens'
    description: Actual number of output tokens.
  - name: num_instances
    display_name: '# eval'
    description: Number of evaluation instances.
  - name: num_train_instances
    display_name: '# train'
    description: Number of training instances (e.g., in-context examples).
  - name: prompt_truncated
    display_name: truncated
    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
  - name: finish_reason_length
    display_name: finish b/c length
    description: Fraction of instances where the the output was terminated because of the max tokens limit.
  - name: finish_reason_stop
    display_name: finish b/c stop
    description: Fraction of instances where the the output was terminated because of the stop sequences.
  - name: finish_reason_endoftext
    display_name: finish b/c endoftext
    description: Fraction of instances where the the output was terminated because the end of text token was generated.
  - name: finish_reason_unknown
    display_name: finish b/c unknown
    description: Fraction of instances where the the output was terminated for unknown reasons.
  # Accuracy metrics:
  - name: exact_match
    display_name: Exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference exactly.
    lower_is_better: false
  - name: quasi_exact_match
    display_name: Quasi-exact match
    short_display_name: EM
    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
    lower_is_better: false
  - name: rouge_1
    display_name: ROUGE-1
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap.
    lower_is_better: false
  - name: rouge_2
    display_name: ROUGE-2
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap.
    lower_is_better: false
  - name: rouge_l
    display_name: ROUGE-L
    description: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap.
    lower_is_better: false
  - name: classification_weighted_f1
    display_name: Weighted F1
    description: Weighted F1 score
    lower_is_better: false
  - name: float_equiv
    display_name: Float Equivalence
    description: Float Equivalence
    lower_is_better: false
  - name: adjusted_macro_f1_score
    display_name: Adjusted Macro F1 Score
    short_display_name: Adjusted Macro F1 Score
    description: Entity type classification F1 score, adjusted for partial matches following the KPI-Edgar paper, macro-averaged across entity types

############################################################
perturbations: []

############################################################
metric_groups:
  - name: accuracy
    display_name: Accuracy
    metrics:
      - name: ${main_name}
        split: ${main_split}

  - name: efficiency
    display_name: Efficiency
    metrics:
    - name: inference_runtime
      split: ${main_split}

  - name: general_information
    display_name: General information
    hide_win_rates: true
    metrics:
    - name: num_instances
      split: ${main_split}
    - name: num_train_instances
      split: ${main_split}
    - name: prompt_truncated
      split: ${main_split}
    - name: num_prompt_tokens
      split: ${main_split}
    - name: num_output_tokens
      split: ${main_split}

############################################################
run_groups:
  - name: financial_scenarios
    display_name: Financial Scenarios
    description: Scenarios for the financial domain
    category: All scenarios
    subgroups:
      - gold_commodity_news
      - financial_phrasebank
      - conv_fin_qa_calc
      - kpi_edgar

  - name: legal_scenarios
    display_name: Legal Scenarios
    description: Scenarios for the legal domain
    category: All scenarios
    subgroups:
      - legal_contract_summarization
      - casehold
      - echr_judgment_classification
      - legal_opinion_sentiment_classification

  - name: climate_scenarios
    display_name: Climate Scenarios
    description: Scenarios for the climate domain
    category: All scenarios
    subgroups:
      - sumosum

  - name: cyber_security_scenarios
    display_name: Cyber Security Scenarios
    description: Scenarios for the cyber security domain
    category: All scenarios
    subgroups:
      - cti_to_mitre

  - name: financial_phrasebank
    display_name: Financial Phrasebank (Sentiment Classification)
    description: A sentiment classification benchmark based on the dataset from Good Debt or Bad Debt - Detecting Semantic Orientations in Economic Texts [(Malo et al., 2013)](https://arxiv.org/abs/1307.5336).
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: classification_weighted_f1
      main_split: test
    taxonomy:
      task: sentiment analysis
      what: phrases from financial news texts and company press releases
      who: annotators with adequate business education background
      when: before 2013
      language: English

  - name: conv_fin_qa_calc
    display_name: ConvFinQACalc
    description: "A mathematical calculation benchmark based on ConvFinQA: Exploring the Chain of Numerical Reasoning in Conversational Finance Question Answering [(Chen ey al., 2022)](https://arxiv.org/pdf/2210.03849.pdf)."
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: float_equiv
      main_split: valid
    taxonomy:
      task: question answering with numeric reasoning
      what: financial reports
      who: financial experts
      when: 1999 to 2019
      language: English

  - name: gold_commodity_news
    display_name: Gold Commodity News
    description: A classification benchmark based on a dataset of human-annotated gold commodity news headlines ([Sinha & Khandait, 2019](https://arxiv.org/abs/2009.04202)).
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: classification_weighted_f1
      main_split: test
    taxonomy:
      task: text classification
      what: gold commodity news headlines
      who: financial journalists
      when: 2000-2019
      language: English

  - name: kpi_edgar
    display_name: KPI-EDGAR Financial Documents (Named Entity Recognition)
    description: A named entity recognition beenchmark based on the paper KPI-EDGAR - A Novel Dataset and Accompanying Metric for Relation Extraction from Financial Documents [(Deußer et al., 2022)](https://arxiv.org/pdf/2210.09163.pdf).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: adjusted_macro_f1_score
      main_split: test
    taxonomy:
      task: named entity recognition
      what: financial reports
      who: financial experts
      when: before 2022
      language: English

  - name: legal_contract_summarization
    display_name: Legal Contract Summarization
    description: Plain English Summarization of Contracts [(Manor et al., 2019)](https://aclanthology.org/W19-2201.pdf).
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: rouge_l
      main_split: test
    taxonomy:
      task: summarization
      what: legal contracts (e.g. terms of service, license agreements)
      who: lawyers
      when: before 2019
      language: English

  - name: casehold
    display_name: CaseHOLD
    description: CaseHOLD (Case Holdings On Legal Decisions) is a multiple choice question answering scenario where the task is to identify the relevant holding of a cited case [(Zheng et al, 2021)](https://arxiv.org/pdf/2104.08671.pdf).
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: quasi_exact_match
      main_split: test
    taxonomy:
      task: question answering
      what: Harvard Law Library case law corpus
      who: legal professionals
      when: before 2021
      language: English

  - name: echr_judgment_classification
    display_name: ECHR Judgment Classification
    description: The "Binary Violation" Classification task from the paper Neural Legal Judgment Prediction in English [(Chalkidis et al., 2019)](https://arxiv.org/pdf/1906.02059.pdf). The task is to analyze the description of a legal case from the European Court of Human Rights (ECHR), and classify it as positive if any human rights article or protocol has been violated and negative otherwise.
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: classification_weighted_f1
      main_split: test
    taxonomy:
      task: text classification
      what: casees from the European Court of Human Rights
      who: judiciary of the European Court of Human Rights
      when: 2014-2018 (train) and 2014-2018 (test)
      language: English

  - name: legal_opinion_sentiment_classification
    display_name: Legal Opinion Sentiment Classification
    description: A legal opinion sentiment classification task based on the paper Effective Approach to Develop a Sentiment Annotator For Legal Domain in a Low Resource Setting [(Ratnayaka et al., 2020)](https://arxiv.org/pdf/2011.00318.pdf).
    metric_groups:
      - accuracy
      - general_information
    environment:
      main_name: quasi_exact_match
      main_split: test
    taxonomy:
      task: sentiment analysis
      what: United States legal opinion texts
      who: United States courts
      when: Before 2020
      language: English

  - name: sumosum
    display_name: SUMO Web Claims Summarization
    description: A summarization benchmark based on the climate subset of the SUMO dataset ([Mishra et al., 2020](https://aclanthology.org/2020.wnut-1.12/)).
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: rouge_l
      main_split: test
    taxonomy:
      task: summarization
      what: Articles from climatefeedback.org
      who: Writers of news articles and web documents
      when: Before 2020
      language: English
      main_name: quasi_exact_match
      main_split: test

  - name: cti_to_mitre
    display_name: CTI-to-MITRE Cyber Threat Intelligence
    description: A classification benchmark based on Automatic Mapping of Unstructured Cyber Threat Intelligence - An Experimental Study [(Orbinato et al., 2022)](https://arxiv.org/pdf/2208.12144.pdf).
    metric_groups:
      - accuracy
      - efficiency
      - general_information
    environment:
      main_name: quasi_exact_match
      main_split: test
    taxonomy:
      task: text classification
      what: Descriptions of malicious techniques
      who: Security professionals
      when: Before 2022
      language: English
