{
    "__type__": "task_card",
    "loader": {
        "__type__": "load_hf",
        "path": "TIGER-Lab/MMLU-Pro",
        "filtering_lambda": "lambda x: x['category'] == 'history'"
    },
    "preprocess_steps": [
        {
            "__type__": "deduplicate",
            "by": [
                "question",
                "options",
                "answer",
                "category"
            ]
        },
        {
            "__type__": "rename_splits",
            "mapper": {
                "validation": "train"
            }
        },
        {
            "__type__": "rename",
            "field_to_field": {
                "options": "choices",
                "answer_index": "answer"
            }
        },
        {
            "__type__": "set",
            "fields": {
                "topic": "history"
            }
        }
    ],
    "task": "tasks.qa.multiple_choice.with_topic",
    "templates": "templates.qa.multiple_choice.with_topic.all",
    "__tags__": {
        "annotations_creators": "no-annotation",
        "arxiv": [
            "2406.01574"
        ],
        "language": "en",
        "language_creators": "expert-generated",
        "license": "mit",
        "multilinguality": "monolingual",
        "size_categories": "10K<n<100K",
        "source_datasets": "original",
        "task_categories": "question-answering",
        "task_ids": "multiple-choice-qa"
    },
    "__description__": "MMLU-Pro dataset is a more robust and challenging massive multi-task understanding dataset tailored to more rigorously benchmark large language models' capabilities. This dataset contains 12K complex questions across various disciplines."
}
