PAPERS={
  "unbabel/wmt23-cometkiwi-da-xl-marian": {
    "url": "https://aclanthology.org/2022.wmt-1.60",
    "citation": "@inproceedings{rei-etal-2022-cometkiwi,\n    title = \"{C}omet{K}iwi: {IST}-Unbabel 2022 Submission for the Quality Estimation Shared Task\",\n    author = \"Rei, Ricardo  and\n      Treviso, Marcos  and\n      Guerreiro, Nuno M.  and\n      Zerva, Chrysoula  and\n      Farinha, Ana C  and\n      Maroti, Christine  and\n      C. de Souza, Jos{\\'e} G.  and\n      Glushkova, Taisiya  and\n      Alves, Duarte  and\n      Coheur, Luisa  and\n      Lavie, Alon  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = {Koehn, Philipp  and\n      Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Freitag, Markus  and\n      Graham, Yvette  and\n      Grundkiewicz, Roman  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Jimeno Yepes, Antonio  and\n      Kocmi, Tom  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo  and\n      N{\\'e}v{\\'e}ol, Aur{\\'e}lie  and\n      Neves, Mariana  and\n      Popel, Martin  and\n      Turchi, Marco  and\n      Zampieri, Marcos},\n    booktitle = \"Proceedings of the Seventh Conference on Machine Translation (WMT)\",\n    month = dec,\n    year = \"2022\",\n    address = \"Abu Dhabi, United Arab Emirates (Hybrid)\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.wmt-1.60\",\n    pages = \"634--645\",\n    abstract = \"We present the joint contribution of IST and Unbabel to the WMT 2022 Shared Task on Quality Estimation (QE). Our team participated in all three subtasks: (i) Sentence and Word-level Quality Prediction; (ii) Explainable QE; and (iii) Critical Error Detection. For all tasks we build on top of the COMET framework, connecting it with the predictor-estimator architecture of OpenKiwi, and equipping it with a word-level sequence tagger and an explanation extractor. Our results suggest that incorporating references during pretraining improves performance across several language pairs on downstream tasks, and that jointly training with sentence and word-level objectives yields a further boost. Furthermore, combining attention and gradient information proved to be the top strategy for extracting good explanations of sentence-level QE models. Overall, our submissions achieved the best results for all three tasks for almost all language pairs by a considerable margin.\",\n}\n"
  },
  "unbabel/wmt21-comet-qe-mqm-marian": {
    "url": "https://aclanthology.org/2021.findings-emnlp.330",
    "citation": "@inproceedings{glushkova-etal-2021-uncertainty-aware,\n    title = \"Uncertainty-Aware Machine Translation Evaluation\",\n    author = \"Glushkova, Taisiya  and\n      Zerva, Chrysoula  and\n      Rei, Ricardo  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = \"Moens, Marie-Francine  and\n      Huang, Xuanjing  and\n      Specia, Lucia  and\n      Yih, Scott Wen-tau\",\n    booktitle = \"Findings of the Association for Computational Linguistics: EMNLP 2021\",\n    month = nov,\n    year = \"2021\",\n    address = \"Punta Cana, Dominican Republic\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2021.findings-emnlp.330\",\n    doi = \"10.18653/v1/2021.findings-emnlp.330\",\n    pages = \"3920--3938\",\n    abstract = \"Several neural-based metrics have been recently proposed to evaluate machine translation quality. However, all of them resort to point estimates, which provide limited information at segment level. This is made worse as they are trained on noisy, biased and scarce human judgements, often resulting in unreliable quality predictions. In this paper, we introduce uncertainty-aware MT evaluation and analyze the trustworthiness of the predicted quality. We combine the COMET framework with two uncertainty estimation methods, Monte Carlo dropout and deep ensembles, to obtain quality scores along with confidence intervals. We compare the performance of our uncertainty-aware MT evaluation methods across multiple language pairs from the QT21 dataset and the WMT20 metrics task, augmented with MQM annotations. We experiment with varying numbers of references and further discuss the usefulness of uncertainty-aware quality estimation (without references) to flag possibly critical translation mistakes.\",\n}\n"
  },
  "unbabel/wmt21-comet-qe-da-marian": {
    "url": "https://aclanthology.org/2021.findings-emnlp.330",
    "citation": "@inproceedings{glushkova-etal-2021-uncertainty-aware,\n    title = \"Uncertainty-Aware Machine Translation Evaluation\",\n    author = \"Glushkova, Taisiya  and\n      Zerva, Chrysoula  and\n      Rei, Ricardo  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = \"Moens, Marie-Francine  and\n      Huang, Xuanjing  and\n      Specia, Lucia  and\n      Yih, Scott Wen-tau\",\n    booktitle = \"Findings of the Association for Computational Linguistics: EMNLP 2021\",\n    month = nov,\n    year = \"2021\",\n    address = \"Punta Cana, Dominican Republic\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2021.findings-emnlp.330\",\n    doi = \"10.18653/v1/2021.findings-emnlp.330\",\n    pages = \"3920--3938\",\n    abstract = \"Several neural-based metrics have been recently proposed to evaluate machine translation quality. However, all of them resort to point estimates, which provide limited information at segment level. This is made worse as they are trained on noisy, biased and scarce human judgements, often resulting in unreliable quality predictions. In this paper, we introduce uncertainty-aware MT evaluation and analyze the trustworthiness of the predicted quality. We combine the COMET framework with two uncertainty estimation methods, Monte Carlo dropout and deep ensembles, to obtain quality scores along with confidence intervals. We compare the performance of our uncertainty-aware MT evaluation methods across multiple language pairs from the QT21 dataset and the WMT20 metrics task, augmented with MQM annotations. We experiment with varying numbers of references and further discuss the usefulness of uncertainty-aware quality estimation (without references) to flag possibly critical translation mistakes.\",\n}\n"
  },
  "unbabel/wmt21-comet-da-marian": {
    "url": "https://aclanthology.org/2021.findings-emnlp.330",
    "citation": "@inproceedings{glushkova-etal-2021-uncertainty-aware,\n    title = \"Uncertainty-Aware Machine Translation Evaluation\",\n    author = \"Glushkova, Taisiya  and\n      Zerva, Chrysoula  and\n      Rei, Ricardo  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = \"Moens, Marie-Francine  and\n      Huang, Xuanjing  and\n      Specia, Lucia  and\n      Yih, Scott Wen-tau\",\n    booktitle = \"Findings of the Association for Computational Linguistics: EMNLP 2021\",\n    month = nov,\n    year = \"2021\",\n    address = \"Punta Cana, Dominican Republic\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2021.findings-emnlp.330\",\n    doi = \"10.18653/v1/2021.findings-emnlp.330\",\n    pages = \"3920--3938\",\n    abstract = \"Several neural-based metrics have been recently proposed to evaluate machine translation quality. However, all of them resort to point estimates, which provide limited information at segment level. This is made worse as they are trained on noisy, biased and scarce human judgements, often resulting in unreliable quality predictions. In this paper, we introduce uncertainty-aware MT evaluation and analyze the trustworthiness of the predicted quality. We combine the COMET framework with two uncertainty estimation methods, Monte Carlo dropout and deep ensembles, to obtain quality scores along with confidence intervals. We compare the performance of our uncertainty-aware MT evaluation methods across multiple language pairs from the QT21 dataset and the WMT20 metrics task, augmented with MQM annotations. We experiment with varying numbers of references and further discuss the usefulness of uncertainty-aware quality estimation (without references) to flag possibly critical translation mistakes.\",\n}\n"
  },
  "unbabel/wmt20-comet-qe-da-v2-marian": {
    "url": "https://aclanthology.org/2020.wmt-1.101",
    "citation": "@inproceedings{rei-etal-2020-unbabels,\n    title = \"Unbabel{'}s Participation in the {WMT}20 Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n    editor = {Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Graham, Yvette  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Yepes, Antonio Jimeno  and\n      Koehn, Philipp  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo},\n    booktitle = \"Proceedings of the Fifth Conference on Machine Translation\",\n    month = nov,\n    year = \"2020\",\n    address = \"Online\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2020.wmt-1.101\",\n    pages = \"911--920\",\n    abstract = \"We present the contribution of the Unbabel team to the WMT 2020 Shared Task on Metrics. We intend to participate on the segmentlevel, document-level and system-level tracks on all language pairs, as well as the {``}QE as a Metric{''} track. Accordingly, we illustrate results of our models in these tracks with reference to test sets from the previous year. Our submissions build upon the recently proposed COMET framework: we train several estimator models to regress on different humangenerated quality scores and a novel ranking model trained on relative ranks obtained from Direct Assessments. We also propose a simple technique for converting segment-level predictions into a document-level score. Overall, our systems achieve strong results for all language pairs on previous test sets and in many cases set a new state-of-the-art.\",\n}\n"
  },
  "unbabel/wmt20-comet-qe-da-marian": {
    "url": "https://aclanthology.org/2020.wmt-1.101",
    "citation": "@inproceedings{rei-etal-2020-unbabels,\n    title = \"Unbabel{'}s Participation in the {WMT}20 Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n    editor = {Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Graham, Yvette  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Yepes, Antonio Jimeno  and\n      Koehn, Philipp  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo},\n    booktitle = \"Proceedings of the Fifth Conference on Machine Translation\",\n    month = nov,\n    year = \"2020\",\n    address = \"Online\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2020.wmt-1.101\",\n    pages = \"911--920\",\n    abstract = \"We present the contribution of the Unbabel team to the WMT 2020 Shared Task on Metrics. We intend to participate on the segmentlevel, document-level and system-level tracks on all language pairs, as well as the {``}QE as a Metric{''} track. Accordingly, we illustrate results of our models in these tracks with reference to test sets from the previous year. Our submissions build upon the recently proposed COMET framework: we train several estimator models to regress on different humangenerated quality scores and a novel ranking model trained on relative ranks obtained from Direct Assessments. We also propose a simple technique for converting segment-level predictions into a document-level score. Overall, our systems achieve strong results for all language pairs on previous test sets and in many cases set a new state-of-the-art.\",\n}\n"
  },
  "unbabel/wmt20-comet-da-marian": {
    "url": "https://aclanthology.org/2020.wmt-1.101",
    "citation": "@inproceedings{rei-etal-2020-unbabels,\n    title = \"Unbabel{'}s Participation in the {WMT}20 Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n    editor = {Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Graham, Yvette  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Yepes, Antonio Jimeno  and\n      Koehn, Philipp  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo},\n    booktitle = \"Proceedings of the Fifth Conference on Machine Translation\",\n    month = nov,\n    year = \"2020\",\n    address = \"Online\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2020.wmt-1.101\",\n    pages = \"911--920\",\n    abstract = \"We present the contribution of the Unbabel team to the WMT 2020 Shared Task on Metrics. We intend to participate on the segmentlevel, document-level and system-level tracks on all language pairs, as well as the {``}QE as a Metric{''} track. Accordingly, we illustrate results of our models in these tracks with reference to test sets from the previous year. Our submissions build upon the recently proposed COMET framework: we train several estimator models to regress on different humangenerated quality scores and a novel ranking model trained on relative ranks obtained from Direct Assessments. We also propose a simple technique for converting segment-level predictions into a document-level score. Overall, our systems achieve strong results for all language pairs on previous test sets and in many cases set a new state-of-the-art.\",\n}\n"
  },
  "unbabel/wmt22-comet-da-marian": {
    "url": "https://aclanthology.org/2022.wmt-1.52",
    "citation": "@inproceedings{rei-etal-2022-comet,\n    title = \"{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      C. de Souza, Jos{\\'e} G.  and\n      Alves, Duarte  and\n      Zerva, Chrysoula  and\n      Farinha, Ana C  and\n      Glushkova, Taisiya  and\n      Lavie, Alon  and\n      Coheur, Luisa  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = {Koehn, Philipp  and\n      Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Freitag, Markus  and\n      Graham, Yvette  and\n      Grundkiewicz, Roman  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Jimeno Yepes, Antonio  and\n      Kocmi, Tom  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo  and\n      N{\\'e}v{\\'e}ol, Aur{\\'e}lie  and\n      Neves, Mariana  and\n      Popel, Martin  and\n      Turchi, Marco  and\n      Zampieri, Marcos},\n    booktitle = \"Proceedings of the Seventh Conference on Machine Translation (WMT)\",\n    month = dec,\n    year = \"2022\",\n    address = \"Abu Dhabi, United Arab Emirates (Hybrid)\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.wmt-1.52\",\n    pages = \"578--585\",\n    abstract = \"In this paper, we present the joint contribution of Unbabel and IST to the WMT 2022 Metrics Shared Task. Our primary submission {--} dubbed COMET-22 {--} is an ensemble between a COMET estimator model trained with Direct Assessments and a newly proposed multitask model trained to predict sentence-level scores along with OK/BAD word-level tags derived from Multidimensional Quality Metrics error annotations. These models are ensembled together using a hyper-parameter search that weights different features extracted from both evaluation models and combines them into a single score. For the reference-free evaluation, we present CometKiwi. Similarly to our primary submission, CometKiwi is an ensemble between two models. A traditional predictor-estimator model inspired by OpenKiwi and our new multitask model trained on Multidimensional Quality Metrics which can also be used without references. Both our submissions show improved correlations compared to state-of-the-art metrics from last year as well as increased robustness to critical errors.\",\n}\n"
  },
  "unbabel/wmt23-cometkiwi-da-xxl-marian": {
    "url": "https://arxiv.org/abs/2309.11925",
    "citation": "@misc{rei2023scalingcometkiwiunbabelist2023,\n      title={Scaling up COMETKIWI: Unbabel-IST 2023 Submission for the Quality Estimation Shared Task}, \n      author={Ricardo Rei and Nuno M. Guerreiro and Jos\u00e9 Pombal and Daan van Stigt and Marcos Treviso and Luisa Coheur and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2023},\n      eprint={2309.11925},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2309.11925}, \n}"
  },
  "unbabel/wmt22-cometkiwi-da-marian": {
    "url": "https://aclanthology.org/2022.wmt-1.60",
    "citation": "@inproceedings{rei-etal-2022-cometkiwi,\n    title = \"{C}omet{K}iwi: {IST}-Unbabel 2022 Submission for the Quality Estimation Shared Task\",\n    author = \"Rei, Ricardo  and\n      Treviso, Marcos  and\n      Guerreiro, Nuno M.  and\n      Zerva, Chrysoula  and\n      Farinha, Ana C  and\n      Maroti, Christine  and\n      C. de Souza, Jos{\\'e} G.  and\n      Glushkova, Taisiya  and\n      Alves, Duarte  and\n      Coheur, Luisa  and\n      Lavie, Alon  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = {Koehn, Philipp  and\n      Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Freitag, Markus  and\n      Graham, Yvette  and\n      Grundkiewicz, Roman  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Jimeno Yepes, Antonio  and\n      Kocmi, Tom  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo  and\n      N{\\'e}v{\\'e}ol, Aur{\\'e}lie  and\n      Neves, Mariana  and\n      Popel, Martin  and\n      Turchi, Marco  and\n      Zampieri, Marcos},\n    booktitle = \"Proceedings of the Seventh Conference on Machine Translation (WMT)\",\n    month = dec,\n    year = \"2022\",\n    address = \"Abu Dhabi, United Arab Emirates (Hybrid)\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.wmt-1.60\",\n    pages = \"634--645\",\n    abstract = \"We present the joint contribution of IST and Unbabel to the WMT 2022 Shared Task on Quality Estimation (QE). Our team participated in all three subtasks: (i) Sentence and Word-level Quality Prediction; (ii) Explainable QE; and (iii) Critical Error Detection. For all tasks we build on top of the COMET framework, connecting it with the predictor-estimator architecture of OpenKiwi, and equipping it with a word-level sequence tagger and an explanation extractor. Our results suggest that incorporating references during pretraining improves performance across several language pairs on downstream tasks, and that jointly training with sentence and word-level objectives yields a further boost. Furthermore, combining attention and gradient information proved to be the top strategy for extracting good explanations of sentence-level QE models. Overall, our submissions achieved the best results for all three tasks for almost all language pairs by a considerable margin.\",\n}\n"
  },
  "unbabel/wmt24-qe-task2-baseline": {
    "url": "https://aclanthology.org/2022.wmt-1.60",
    "citation": "@inproceedings{rei-etal-2022-cometkiwi,\n    title = \"{C}omet{K}iwi: {IST}-Unbabel 2022 Submission for the Quality Estimation Shared Task\",\n    author = \"Rei, Ricardo  and\n      Treviso, Marcos  and\n      Guerreiro, Nuno M.  and\n      Zerva, Chrysoula  and\n      Farinha, Ana C  and\n      Maroti, Christine  and\n      C. de Souza, Jos{\\'e} G.  and\n      Glushkova, Taisiya  and\n      Alves, Duarte  and\n      Coheur, Luisa  and\n      Lavie, Alon  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = {Koehn, Philipp  and\n      Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Freitag, Markus  and\n      Graham, Yvette  and\n      Grundkiewicz, Roman  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Jimeno Yepes, Antonio  and\n      Kocmi, Tom  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo  and\n      N{\\'e}v{\\'e}ol, Aur{\\'e}lie  and\n      Neves, Mariana  and\n      Popel, Martin  and\n      Turchi, Marco  and\n      Zampieri, Marcos},\n    booktitle = \"Proceedings of the Seventh Conference on Machine Translation (WMT)\",\n    month = dec,\n    year = \"2022\",\n    address = \"Abu Dhabi, United Arab Emirates (Hybrid)\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.wmt-1.60\",\n    pages = \"634--645\",\n    abstract = \"We present the joint contribution of IST and Unbabel to the WMT 2022 Shared Task on Quality Estimation (QE). Our team participated in all three subtasks: (i) Sentence and Word-level Quality Prediction; (ii) Explainable QE; and (iii) Critical Error Detection. For all tasks we build on top of the COMET framework, connecting it with the predictor-estimator architecture of OpenKiwi, and equipping it with a word-level sequence tagger and an explanation extractor. Our results suggest that incorporating references during pretraining improves performance across several language pairs on downstream tasks, and that jointly training with sentence and word-level objectives yields a further boost. Furthermore, combining attention and gradient information proved to be the top strategy for extracting good explanations of sentence-level QE models. Overall, our submissions achieved the best results for all three tasks for almost all language pairs by a considerable margin.\",\n}\n"
  },
  "unbabel/wmt22-cometkiwi-da": {
    "url": "https://aclanthology.org/2022.wmt-1.60",
    "citation": "@inproceedings{rei-etal-2022-cometkiwi,\n    title = \"{C}omet{K}iwi: {IST}-Unbabel 2022 Submission for the Quality Estimation Shared Task\",\n    author = \"Rei, Ricardo  and\n      Treviso, Marcos  and\n      Guerreiro, Nuno M.  and\n      Zerva, Chrysoula  and\n      Farinha, Ana C  and\n      Maroti, Christine  and\n      C. de Souza, Jos{\\'e} G.  and\n      Glushkova, Taisiya  and\n      Alves, Duarte  and\n      Coheur, Luisa  and\n      Lavie, Alon  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = {Koehn, Philipp  and\n      Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Freitag, Markus  and\n      Graham, Yvette  and\n      Grundkiewicz, Roman  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Jimeno Yepes, Antonio  and\n      Kocmi, Tom  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo  and\n      N{\\'e}v{\\'e}ol, Aur{\\'e}lie  and\n      Neves, Mariana  and\n      Popel, Martin  and\n      Turchi, Marco  and\n      Zampieri, Marcos},\n    booktitle = \"Proceedings of the Seventh Conference on Machine Translation (WMT)\",\n    month = dec,\n    year = \"2022\",\n    address = \"Abu Dhabi, United Arab Emirates (Hybrid)\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.wmt-1.60\",\n    pages = \"634--645\",\n    abstract = \"We present the joint contribution of IST and Unbabel to the WMT 2022 Shared Task on Quality Estimation (QE). Our team participated in all three subtasks: (i) Sentence and Word-level Quality Prediction; (ii) Explainable QE; and (iii) Critical Error Detection. For all tasks we build on top of the COMET framework, connecting it with the predictor-estimator architecture of OpenKiwi, and equipping it with a word-level sequence tagger and an explanation extractor. Our results suggest that incorporating references during pretraining improves performance across several language pairs on downstream tasks, and that jointly training with sentence and word-level objectives yields a further boost. Furthermore, combining attention and gradient information proved to be the top strategy for extracting good explanations of sentence-level QE models. Overall, our submissions achieved the best results for all three tasks for almost all language pairs by a considerable margin.\",\n}\n"
  },
  "unbabel/xcomet-xl": {
    "url": "https://arxiv.org/abs/2310.10482",
    "citation": "@misc{guerreiro2023xcomettransparentmachinetranslation,\n      title={xCOMET: Transparent Machine Translation Evaluation through Fine-grained Error Detection}, \n      author={Nuno M. Guerreiro and Ricardo Rei and Daan van Stigt and Luisa Coheur and Pierre Colombo and Andr\u00e9 F. T. Martins},\n      year={2023},\n      eprint={2310.10482},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2310.10482}, \n}"
  },
  "unbabel/xcomet-xxl": {
    "url": "https://arxiv.org/abs/2310.10482",
    "citation": "@misc{guerreiro2023xcomettransparentmachinetranslation,\n      title={xCOMET: Transparent Machine Translation Evaluation through Fine-grained Error Detection}, \n      author={Nuno M. Guerreiro and Ricardo Rei and Daan van Stigt and Luisa Coheur and Pierre Colombo and Andr\u00e9 F. T. Martins},\n      year={2023},\n      eprint={2310.10482},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2310.10482}, \n}"
  },
  "unbabel/towerinstruct-13b-v0.1": {
    "url": "https://arxiv.org/abs/2402.17733",
    "citation": "@misc{alves2024toweropenmultilinguallarge,\n      title={Tower: An Open Multilingual Large Language Model for Translation-Related Tasks}, \n      author={Duarte M. Alves and Jos\u00e9 Pombal and Nuno M. Guerreiro and Pedro H. Martins and Jo\u00e3o Alves and Amin Farajian and Ben Peters and Ricardo Rei and Patrick Fernandes and Sweta Agrawal and Pierre Colombo and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2024},\n      eprint={2402.17733},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2402.17733}, \n}"
  },
  "unbabel/towerinstruct-7b-v0.2": {
    "url": "https://arxiv.org/abs/2402.17733",
    "citation": "@misc{alves2024toweropenmultilinguallarge,\n      title={Tower: An Open Multilingual Large Language Model for Translation-Related Tasks}, \n      author={Duarte M. Alves and Jos\u00e9 Pombal and Nuno M. Guerreiro and Pedro H. Martins and Jo\u00e3o Alves and Amin Farajian and Ben Peters and Ricardo Rei and Patrick Fernandes and Sweta Agrawal and Pierre Colombo and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2024},\n      eprint={2402.17733},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2402.17733}, \n}"
  },
  "unbabel/towerbase-7b-v0.1": {
    "url": "https://arxiv.org/abs/2402.17733",
    "citation": "@misc{alves2024toweropenmultilinguallarge,\n      title={Tower: An Open Multilingual Large Language Model for Translation-Related Tasks}, \n      author={Duarte M. Alves and Jos\u00e9 Pombal and Nuno M. Guerreiro and Pedro H. Martins and Jo\u00e3o Alves and Amin Farajian and Ben Peters and Ricardo Rei and Patrick Fernandes and Sweta Agrawal and Pierre Colombo and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2024},\n      eprint={2402.17733},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2402.17733}, \n}"
  },
  "unbabel/towerbase-13b-v0.1": {
    "url": "https://arxiv.org/abs/2402.17733",
    "citation": "@misc{alves2024toweropenmultilinguallarge,\n      title={Tower: An Open Multilingual Large Language Model for Translation-Related Tasks}, \n      author={Duarte M. Alves and Jos\u00e9 Pombal and Nuno M. Guerreiro and Pedro H. Martins and Jo\u00e3o Alves and Amin Farajian and Ben Peters and Ricardo Rei and Patrick Fernandes and Sweta Agrawal and Pierre Colombo and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2024},\n      eprint={2402.17733},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2402.17733}, \n}"
  },
  "unbabel/towerinstruct-7b-v0.1": {
    "url": "https://arxiv.org/abs/2402.17733",
    "citation": "@misc{alves2024toweropenmultilinguallarge,\n      title={Tower: An Open Multilingual Large Language Model for Translation-Related Tasks}, \n      author={Duarte M. Alves and Jos\u00e9 Pombal and Nuno M. Guerreiro and Pedro H. Martins and Jo\u00e3o Alves and Amin Farajian and Ben Peters and Ricardo Rei and Patrick Fernandes and Sweta Agrawal and Pierre Colombo and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2024},\n      eprint={2402.17733},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2402.17733}, \n}"
  },
  "unbabel/unite-xl": {
    "url": "https://aclanthology.org/2022.acl-long.558",
    "citation": "@inproceedings{wan-etal-2022-unite,\n    title = \"{U}ni{TE}: Unified Translation Evaluation\",\n    author = \"Wan, Yu  and\n      Liu, Dayiheng  and\n      Yang, Baosong  and\n      Zhang, Haibo  and\n      Chen, Boxing  and\n      Wong, Derek  and\n      Chao, Lidia\",\n    editor = \"Muresan, Smaranda  and\n      Nakov, Preslav  and\n      Villavicencio, Aline\",\n    booktitle = \"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)\",\n    month = may,\n    year = \"2022\",\n    address = \"Dublin, Ireland\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.acl-long.558\",\n    doi = \"10.18653/v1/2022.acl-long.558\",\n    pages = \"8117--8127\",\n    abstract = \"Translation quality evaluation plays a crucial role in machine translation. According to the input format, it is mainly separated into three tasks, \\textit{i.e.}, reference-only, source-only and source-reference-combined. Recent methods, despite their promising results, are specifically designed and optimized on one of them. This limits the convenience of these methods, and overlooks the commonalities among tasks. In this paper, we propose , which is the first unified framework engaged with abilities to handle all three evaluation tasks. Concretely, we propose monotonic regional attention to control the interaction among input segments, and unified pretraining to better adapt multi-task training. We testify our framework on WMT 2019 Metrics and WMT 2020 Quality Estimation benchmarks. Extensive analyses show that our \\textit{single model} can universally surpass various state-of-the-art or winner methods across tasks.Both source code and associated models are available at \\url{https://github.com/NLP2CT/UniTE}.\",\n}\n"
  },
  "unbabel/unite-xxl": {
    "url": "https://aclanthology.org/2022.acl-long.558",
    "citation": "@inproceedings{wan-etal-2022-unite,\n    title = \"{U}ni{TE}: Unified Translation Evaluation\",\n    author = \"Wan, Yu  and\n      Liu, Dayiheng  and\n      Yang, Baosong  and\n      Zhang, Haibo  and\n      Chen, Boxing  and\n      Wong, Derek  and\n      Chao, Lidia\",\n    editor = \"Muresan, Smaranda  and\n      Nakov, Preslav  and\n      Villavicencio, Aline\",\n    booktitle = \"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)\",\n    month = may,\n    year = \"2022\",\n    address = \"Dublin, Ireland\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.acl-long.558\",\n    doi = \"10.18653/v1/2022.acl-long.558\",\n    pages = \"8117--8127\",\n    abstract = \"Translation quality evaluation plays a crucial role in machine translation. According to the input format, it is mainly separated into three tasks, \\textit{i.e.}, reference-only, source-only and source-reference-combined. Recent methods, despite their promising results, are specifically designed and optimized on one of them. This limits the convenience of these methods, and overlooks the commonalities among tasks. In this paper, we propose , which is the first unified framework engaged with abilities to handle all three evaluation tasks. Concretely, we propose monotonic regional attention to control the interaction among input segments, and unified pretraining to better adapt multi-task training. We testify our framework on WMT 2019 Metrics and WMT 2020 Quality Estimation benchmarks. Extensive analyses show that our \\textit{single model} can universally surpass various state-of-the-art or winner methods across tasks.Both source code and associated models are available at \\url{https://github.com/NLP2CT/UniTE}.\",\n}\n"
  },
  "unbabel/unite-mup": {
    "url": "https://aclanthology.org/2022.acl-long.558",
    "citation": "@inproceedings{wan-etal-2022-unite,\n    title = \"{U}ni{TE}: Unified Translation Evaluation\",\n    author = \"Wan, Yu  and\n      Liu, Dayiheng  and\n      Yang, Baosong  and\n      Zhang, Haibo  and\n      Chen, Boxing  and\n      Wong, Derek  and\n      Chao, Lidia\",\n    editor = \"Muresan, Smaranda  and\n      Nakov, Preslav  and\n      Villavicencio, Aline\",\n    booktitle = \"Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)\",\n    month = may,\n    year = \"2022\",\n    address = \"Dublin, Ireland\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.acl-long.558\",\n    doi = \"10.18653/v1/2022.acl-long.558\",\n    pages = \"8117--8127\",\n    abstract = \"Translation quality evaluation plays a crucial role in machine translation. According to the input format, it is mainly separated into three tasks, \\textit{i.e.}, reference-only, source-only and source-reference-combined. Recent methods, despite their promising results, are specifically designed and optimized on one of them. This limits the convenience of these methods, and overlooks the commonalities among tasks. In this paper, we propose , which is the first unified framework engaged with abilities to handle all three evaluation tasks. Concretely, we propose monotonic regional attention to control the interaction among input segments, and unified pretraining to better adapt multi-task training. We testify our framework on WMT 2019 Metrics and WMT 2020 Quality Estimation benchmarks. Extensive analyses show that our \\textit{single model} can universally surpass various state-of-the-art or winner methods across tasks.Both source code and associated models are available at \\url{https://github.com/NLP2CT/UniTE}.\",\n}\n"
  },
  "unbabel/wmt23-cometkiwi-da-xl": {
    "url": "https://arxiv.org/abs/2309.11925",
    "citation": "@misc{rei2023scalingcometkiwiunbabelist2023,\n      title={Scaling up COMETKIWI: Unbabel-IST 2023 Submission for the Quality Estimation Shared Task}, \n      author={Ricardo Rei and Nuno M. Guerreiro and Jos\u00e9 Pombal and Daan van Stigt and Marcos Treviso and Luisa Coheur and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2023},\n      eprint={2309.11925},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2309.11925}, \n}"
  },
  "unbabel/wmt23-cometkiwi-da-xxl": {
    "url": "https://arxiv.org/abs/2309.11925",
    "citation": "@misc{rei2023scalingcometkiwiunbabelist2023,\n      title={Scaling up COMETKIWI: Unbabel-IST 2023 Submission for the Quality Estimation Shared Task}, \n      author={Ricardo Rei and Nuno M. Guerreiro and Jos\u00e9 Pombal and Daan van Stigt and Marcos Treviso and Luisa Coheur and Jos\u00e9 G. C. de Souza and Andr\u00e9 F. T. Martins},\n      year={2023},\n      eprint={2309.11925},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2309.11925}, \n}"
  },
  "unbabel/wmt22-unite-da": {
    "url": "https://arxiv.org/abs/2305.11806",
    "citation": "@misc{rei2023insidestorybetterunderstanding,\n      title={The Inside Story: Towards Better Understanding of Machine Translation Neural Evaluation Metrics}, \n      author={Ricardo Rei and Nuno M. Guerreiro and Marcos Treviso and Luisa Coheur and Alon Lavie and Andr\u00e9 F. T. Martins},\n      year={2023},\n      eprint={2305.11806},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2305.11806}, \n}"
  },
  "unbabel/eamt22-cometinho-da": {
    "url": "https://aclanthology.org/2022.eamt-1.9",
    "citation": "@inproceedings{rei-etal-2022-searching,\n    title = \"Searching for {COMETINHO}: The Little Metric That Could\",\n    author = \"Rei, Ricardo  and\n      Farinha, Ana C  and\n      de Souza, Jos{\\'e} G.C.  and\n      Ramos, Pedro G.  and\n      Martins, Andr{\\'e} F.T.  and\n      Coheur, Luisa  and\n      Lavie, Alon\",\n    editor = {Moniz, Helena  and\n      Macken, Lieve  and\n      Rufener, Andrew  and\n      Barrault, Lo{\\\"\\i}c  and\n      Costa-juss{\\`a}, Marta R.  and\n      Declercq, Christophe  and\n      Koponen, Maarit  and\n      Kemp, Ellie  and\n      Pilos, Spyridon  and\n      Forcada, Mikel L.  and\n      Scarton, Carolina  and\n      Van den Bogaert, Joachim  and\n      Daems, Joke  and\n      Tezcan, Arda  and\n      Vanroy, Bram  and\n      Fonteyne, Margot},\n    booktitle = \"Proceedings of the 23rd Annual Conference of the European Association for Machine Translation\",\n    month = jun,\n    year = \"2022\",\n    address = \"Ghent, Belgium\",\n    publisher = \"European Association for Machine Translation\",\n    url = \"https://aclanthology.org/2022.eamt-1.9\",\n    pages = \"61--70\",\n    abstract = \"In recent years, several neural fine-tuned machine translation evaluation metrics such as COMET and BLEURT have been proposed. These metrics achieve much higher correlations with human judgments than lexical overlap metrics at the cost of computational efficiency and simplicity, limiting their applications to scenarios in which one has to score thousands of translation hypothesis (e.g. scoring multiple systems or Minimum Bayes Risk decoding). In this paper, we explore optimization techniques, pruning, and knowledge distillation to create more compact and faster COMET versions. Our results show that just by optimizing the code through the use of caching and length batching we can reduce inference time between 39{\\%} and 65{\\%} when scoring multiple systems. Also, we show that pruning COMET can lead to a 21{\\%} model reduction without affecting the model{'}s accuracy beyond 0.01 Kendall tau correlation. Furthermore, we present DISTIL-COMET a lightweight distilled version that is 80{\\%} smaller and 2.128x faster while attaining a performance close to the original model and above strong baselines such as BERTSCORE and PRISM.\",\n}\n"
  },
  "unbabel/wmt20-comet-da": {
    "url": "https://aclanthology.org/2020.wmt-1.101",
    "citation": "@inproceedings{rei-etal-2020-unbabels,\n    title = \"Unbabel{'}s Participation in the {WMT}20 Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n    editor = {Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Graham, Yvette  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Yepes, Antonio Jimeno  and\n      Koehn, Philipp  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo},\n    booktitle = \"Proceedings of the Fifth Conference on Machine Translation\",\n    month = nov,\n    year = \"2020\",\n    address = \"Online\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2020.wmt-1.101\",\n    pages = \"911--920\",\n    abstract = \"We present the contribution of the Unbabel team to the WMT 2020 Shared Task on Metrics. We intend to participate on the segmentlevel, document-level and system-level tracks on all language pairs, as well as the {``}QE as a Metric{''} track. Accordingly, we illustrate results of our models in these tracks with reference to test sets from the previous year. Our submissions build upon the recently proposed COMET framework: we train several estimator models to regress on different humangenerated quality scores and a novel ranking model trained on relative ranks obtained from Direct Assessments. We also propose a simple technique for converting segment-level predictions into a document-level score. Overall, our systems achieve strong results for all language pairs on previous test sets and in many cases set a new state-of-the-art.\",\n}\n"
  },
  "unbabel/wmt20-comet-qe-da": {
    "url": "https://aclanthology.org/2020.wmt-1.101",
    "citation": "@inproceedings{rei-etal-2020-unbabels,\n    title = \"Unbabel{'}s Participation in the {WMT}20 Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n    editor = {Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Graham, Yvette  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Yepes, Antonio Jimeno  and\n      Koehn, Philipp  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo},\n    booktitle = \"Proceedings of the Fifth Conference on Machine Translation\",\n    month = nov,\n    year = \"2020\",\n    address = \"Online\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2020.wmt-1.101\",\n    pages = \"911--920\",\n    abstract = \"We present the contribution of the Unbabel team to the WMT 2020 Shared Task on Metrics. We intend to participate on the segmentlevel, document-level and system-level tracks on all language pairs, as well as the {``}QE as a Metric{''} track. Accordingly, we illustrate results of our models in these tracks with reference to test sets from the previous year. Our submissions build upon the recently proposed COMET framework: we train several estimator models to regress on different humangenerated quality scores and a novel ranking model trained on relative ranks obtained from Direct Assessments. We also propose a simple technique for converting segment-level predictions into a document-level score. Overall, our systems achieve strong results for all language pairs on previous test sets and in many cases set a new state-of-the-art.\",\n}\n"
  },
  "unbabel/wmt22-comet-da": {
    "url": "https://aclanthology.org/2022.wmt-1.52",
    "citation": "@inproceedings{rei-etal-2022-comet,\n    title = \"{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task\",\n    author = \"Rei, Ricardo  and\n      C. de Souza, Jos{\\'e} G.  and\n      Alves, Duarte  and\n      Zerva, Chrysoula  and\n      Farinha, Ana C  and\n      Glushkova, Taisiya  and\n      Lavie, Alon  and\n      Coheur, Luisa  and\n      Martins, Andr{\\'e} F. T.\",\n    editor = {Koehn, Philipp  and\n      Barrault, Lo{\\\"\\i}c  and\n      Bojar, Ond{\\v{r}}ej  and\n      Bougares, Fethi  and\n      Chatterjee, Rajen  and\n      Costa-juss{\\`a}, Marta R.  and\n      Federmann, Christian  and\n      Fishel, Mark  and\n      Fraser, Alexander  and\n      Freitag, Markus  and\n      Graham, Yvette  and\n      Grundkiewicz, Roman  and\n      Guzman, Paco  and\n      Haddow, Barry  and\n      Huck, Matthias  and\n      Jimeno Yepes, Antonio  and\n      Kocmi, Tom  and\n      Martins, Andr{\\'e}  and\n      Morishita, Makoto  and\n      Monz, Christof  and\n      Nagata, Masaaki  and\n      Nakazawa, Toshiaki  and\n      Negri, Matteo  and\n      N{\\'e}v{\\'e}ol, Aur{\\'e}lie  and\n      Neves, Mariana  and\n      Popel, Martin  and\n      Turchi, Marco  and\n      Zampieri, Marcos},\n    booktitle = \"Proceedings of the Seventh Conference on Machine Translation (WMT)\",\n    month = dec,\n    year = \"2022\",\n    address = \"Abu Dhabi, United Arab Emirates (Hybrid)\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2022.wmt-1.52\",\n    pages = \"578--585\",\n    abstract = \"In this paper, we present the joint contribution of Unbabel and IST to the WMT 2022 Metrics Shared Task. Our primary submission {--} dubbed COMET-22 {--} is an ensemble between a COMET estimator model trained with Direct Assessments and a newly proposed multitask model trained to predict sentence-level scores along with OK/BAD word-level tags derived from Multidimensional Quality Metrics error annotations. These models are ensembled together using a hyper-parameter search that weights different features extracted from both evaluation models and combines them into a single score. For the reference-free evaluation, we present CometKiwi. Similarly to our primary submission, CometKiwi is an ensemble between two models. A traditional predictor-estimator model inspired by OpenKiwi and our new multitask model trained on Multidimensional Quality Metrics which can also be used without references. Both our submissions show improved correlations compared to state-of-the-art metrics from last year as well as increased robustness to critical errors.\",\n}\n"
  },
  "default": {
    "url": "https://aclanthology.org/2020.emnlp-main.213",
    "citation": "@inproceedings{rei-etal-2020-comet,\n    title = \"{COMET}: A Neural Framework for {MT} Evaluation\",\n    author = \"Rei, Ricardo  and\n      Stewart, Craig  and\n      Farinha, Ana C  and\n      Lavie, Alon\",\n    editor = \"Webber, Bonnie  and\n      Cohn, Trevor  and\n      He, Yulan  and\n      Liu, Yang\",\n    booktitle = \"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)\",\n    month = nov,\n    year = \"2020\",\n    address = \"Online\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2020.emnlp-main.213\",\n    doi = \"10.18653/v1/2020.emnlp-main.213\",\n    pages = \"2685--2702\",\n    abstract = \"We present COMET, a neural framework for training multilingual machine translation evaluation models which obtains new state-of-the-art levels of correlation with human judgements. Our framework leverages recent breakthroughs in cross-lingual pretrained language modeling resulting in highly multilingual and adaptable MT evaluation models that exploit information from both the source input and a target-language reference translation in order to more accurately predict MT quality. To showcase our framework, we train three models with different types of human judgements: Direct Assessments, Human-mediated Translation Edit Rate and Multidimensional Quality Metric. Our models achieve new state-of-the-art performance on the WMT 2019 Metrics shared task and demonstrate robustness to high-performing systems.\",\n}\n"
  }
}