{
    "__type__": "task_card",
    "loader": {
        "__type__": "load_hf",
        "path": "MMMU/MMMU",
        "name": "History",
        "data_classification_policy": [
            "public"
        ]
    },
    "preprocess_steps": [
        {
            "__type__": "rename_splits",
            "mapper": {
                "dev": "train",
                "validation": "test"
            }
        },
        {
            "__type__": "list_field_values",
            "fields": [
                "image_1",
                "image_2",
                "image_3",
                "image_4",
                "image_5",
                "image_6",
                "image_7"
            ],
            "to_field": "media/images"
        },
        {
            "__type__": "filter",
            "field": "media/images",
            "values": [
                null
            ]
        },
        {
            "__type__": "map_replace",
            "field_to_field": {
                "question": "question",
                "options": "choices"
            },
            "mapping": {
                "<image 1>": "<img src=\"media/images/0\">",
                "<image 2>": "<img src=\"media/images/1\">",
                "<image 3>": "<img src=\"media/images/2\">",
                "<image 4>": "<img src=\"media/images/3\">",
                "<image 5>": "<img src=\"media/images/4\">",
                "<image 6>": "<img src=\"media/images/5\">",
                "<image 7>": "<img src=\"media/images/6\">"
            }
        },
        {
            "__type__": "literal_eval",
            "field": "choices"
        },
        {
            "__type__": "lower",
            "field": "subfield",
            "to_field": "topic"
        },
        {
            "__type__": "map_values",
            "field": "answer",
            "mapping": {
                "A": 0,
                "B": 1,
                "C": 2,
                "D": 3,
                "E": 4,
                "?": null
            }
        }
    ],
    "task": "tasks.qa.multiple_choice.with_topic",
    "templates": "templates.qa.multiple_choice.with_topic.all",
    "__tags__": {
        "language": [
            "en"
        ],
        "license": "apache-2.0",
        "size_categories": [
            "10K<n<100K"
        ],
        "task_categories": [
            "question-answering",
            "visual-question-answering",
            "multiple-choice"
        ]
    },
    "__description__": "MMMU: a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning. MMMU includes 11.5K meticulously collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering. These questions span 30 subjects and 183 subfields, comprising 30 highly heterogeneous image types, such as charts, diagrams, maps, tables, music sheets, and chemical structures. We believe MMMU will stimulate the community to build next-generation multimodal foundation models towards expert artificial general intelligence (AGI)."
}
