from mteb.abstasks.image.image_text_pair_classification import (
    AbsTaskImageTextPairClassification,
)
from mteb.abstasks.task_metadata import TaskMetadata


class AROVisualRelation(AbsTaskImageTextPairClassification):
    images_column_names = ("image",)
    texts_column_names = ("true_caption", "false_caption")

    metadata = TaskMetadata(
        name="AROVisualRelation",
        description="Compositionality Evaluation of images to their captions.",
        reference="https://openreview.net/forum?id=KRLUvxh8uaX",
        dataset={
            "path": "mteb/ARO-Visual-Relation",
            "revision": "53c1115ab5871ab33b09d5bb8856236d46bdaf2d",
        },
        type="Compositionality",
        category="i2t",
        eval_splits=["test"],
        eval_langs=["eng-Latn"],
        main_score="text_acc",
        date=(
            "2022-01-01",
            "2022-12-31",
        ),  # Estimated range for the collection of data
        domains=["Encyclopaedic"],
        task_subtypes=["Caption Pairing"],
        license="mit",
        annotations_creators="expert-annotated",
        dialect=[],
        modalities=["text", "image"],
        sample_creation="created",
        bibtex_citation=r"""
@inproceedings{yuksekgonul2023and,
  author = {Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James},
  booktitle = {The Eleventh International Conference on Learning Representations},
  title = {When and why vision-language models behave like bags-of-words, and what to do about it?},
  year = {2023},
}
""",
    )
