from __future__ import annotations

from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
from mteb.abstasks.TaskMetadata import TaskMetadata


class ChemHotpotQARetrieval(AbsTaskRetrieval):
    metadata = TaskMetadata(
        name="ChemHotpotQARetrieval",
        dataset={
            "path": "BASF-AI/ChemHotpotQARetrieval",
            "revision": "1840e8a5ac6ec752bbdd97d543ead0189bc7c25b",
        },
        description="ChemTEB evaluates the performance of text embedding models on chemical domain data.",
        reference="https://arxiv.org/abs/2412.00532",
        type="Retrieval",
        category="s2p",
        modalities=["text"],
        eval_splits=["train", "dev", "test"],
        eval_langs=["eng-Latn"],
        main_score="ndcg_at_10",
        date=("2024-06-01", "2024-11-30"),
        domains=["Chemistry"],
        task_subtypes=[],
        license="cc-by-nc-sa-4.0",
        annotations_creators="derived",
        dialect=[],
        sample_creation="found",
        bibtex_citation="""
        @article{kasmaee2024chemteb,
        title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain},
        author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila},
        journal={arXiv preprint arXiv:2412.00532},
        year={2024}
        }
        @inproceedings{yang-etal-2018-hotpotqa,
            title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering",
            author = "Yang, Zhilin  and
            Qi, Peng  and
            Zhang, Saizheng  and
            Bengio, Yoshua  and
            Cohen, William  and
            Salakhutdinov, Ruslan  and
            Manning, Christopher D.",
            editor = "Riloff, Ellen  and
            Chiang, David  and
            Hockenmaier, Julia  and
            Tsujii, Jun{'}ichi",
            booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing",
            month = oct # "-" # nov,
            year = "2018",
            address = "Brussels, Belgium",
            publisher = "Association for Computational Linguistics",
            url = "https://aclanthology.org/D18-1259",
            doi = "10.18653/v1/D18-1259",
            pages = "2369--2380",
            abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.",
        }
""",
    )
