"""Noreval group task manifest."""

from __future__ import annotations

BASE_IMPORT = "wisent.core.contrastive_pairs.lm_eval_pairs.lm_task_extractors."

# NO PARENT TASK MAPPING - each subtask mapped individually based on output_type

NOREVAL_TASKS = {
    # Generate_until subtasks → noreval_gen.py (evaluator_name = "generation")
    # ask_gec
    "ask_gec_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "ask_gec_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "ask_gec_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "ask_gec_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "ask_gec_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    # noridiom
    "noridiom_nno_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nno_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nno_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nno_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nno_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nob_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nob_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nob_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nob_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "noridiom_nob_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    # norquad
    "norquad_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norquad_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norquad_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norquad_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norquad_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    # norrewrite/norsummarize instruct
    "norrewrite_instruct": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsummarize_instruct": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    # norsumm
    "norsumm_nno_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nno_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nno_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nno_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nno_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nno_p5": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nob_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nob_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nob_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nob_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nob_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "norsumm_nob_p5": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    # nortruthfulqa_gen
    "nortruthfulqa_gen_nno_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nno_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nno_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nno_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nno_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nob_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nob_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nob_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nob_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "nortruthfulqa_gen_nob_p4": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    # tatoeba
    "tatoeba_eng_nno_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nno_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nno_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nno_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nob_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nob_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nob_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_eng_nob_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nno_eng_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nno_eng_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nno_eng_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nno_eng_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nob_eng_p0": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nob_eng_p1": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nob_eng_p2": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",
    "tatoeba_nob_eng_p3": f"{BASE_IMPORT}noreval_gen:NorevalGenerationExtractor",

    # Multiple_choice subtasks → noreval_mc.py (evaluator_name = "log_likelihoods")
    # ncb
    "ncb": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    # norbelebele
    "norbelebele_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norbelebele_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norbelebele_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norbelebele_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norbelebele_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    # norcommonsenseqa
    "norcommonsenseqa_nno_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nno_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nno_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nno_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nno_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nob_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nob_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nob_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nob_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norcommonsenseqa_nob_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    # norec
    "norec_document_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_document_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_document_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_document_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_document_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_sentence_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_sentence_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_sentence_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_sentence_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "norec_sentence_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    # noropenbookqa
    "noropenbookqa_nno_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nno_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nno_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nno_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nno_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nob_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nob_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nob_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nob_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "noropenbookqa_nob_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    # nortruthfulqa_mc
    "nortruthfulqa_mc_nno_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nno_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nno_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nno_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nno_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nob_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nob_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nob_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nob_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nortruthfulqa_mc_nob_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    # nrk_quiz_qa
    "nrk_quiz_qa_nno_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nno_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nno_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nno_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nno_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nob_p0": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nob_p1": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nob_p2": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nob_p3": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
    "nrk_quiz_qa_nob_p4": f"{BASE_IMPORT}noreval_mc:NorevalMultipleChoiceExtractor",
}
