#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (c) Jason Young (杨郑鑫).
#
# E-Mail: <AI.Jason.Young@outlook.com>
# 2024-04-14 09:22
#
# This source code is licensed under the Apache-2.0 license found in the
# LICENSE file in the root directory of this source tree.


from younger.datasets.constructors.huggingface.annotations import parse_dataset, parse_metric, normalize_metric_value

from younger.datasets.constructors.huggingface.annotations import get_heuristic_annotations
from younger.datasets.constructors.huggingface.utils import infer_model_size, clean_default_cache_repo, clean_specify_cache_repo, get_huggingface_model_readme, get_huggingface_model_card_data_from_readme
from huggingface_hub import HfFileSystem

hf_file_system = HfFileSystem()
model_ids = [
  # 'lmqg/t5-small-squad-qg',
  # 'luisotorres/bart-finetuned-samsum',
  'lamhieu/ghost-7b-v0.9.0',
  # 'Zabihin/Symptom_to_Diagnosis',
  # 'Jzuluaga/bert-base-ner-atc-en-atco2-1h',
  # 'tarek23/flan-t5-qg-tarek-test-SQUAD',
  # 'aXhyra/presentation_sentiment_1234567',
  # 'Harveenchadha/hindi_base_wav2vec2',
  # 'azizbarank/mbert-finetuned-azerbaijani-ner',
  # 'DataGuard/pali-7B-v0.16.4-awq',
  # 'it5/it5-efficient-small-el32-news-summarization',
  # 'TheBloke/bloomz-176B-GPTQ',
  # 'thomas0104/whisper_medium_nan_tw',
  # 'bigscience/bloom-3b',
  # 'BallisticAI/Ballistic-CodeLlama-34B-v1-AWQ',
  # 'sysresearch101/t5-large-xsum-cnn-8-2',
  # 'research-backup/relbert-roberta-large-triplet-e-semeval2012',
  # 'Aditya3107/wav2vec2-Irish-common-voice-Fleurs-living-audio-300m',
]
for model_id in model_ids:
  print(f'-------------------------{model_id}----------------------------')
  readme = get_huggingface_model_readme(model_id, hf_file_system)
  cd = get_huggingface_model_card_data_from_readme(readme)
  labels = get_heuristic_annotations(model_id, cd)
  for label in labels:
    print(label)

# x = 'qwk'
# metric = parse_metric(x)
# print(normalize_metric_value(metric, '0.5'))

# ii = [
#   "cos_sim_f1",
#   "auc",
#   "map_at_5",
#   "test rouge2",
#   "diffbleu",
#   "kurmanji test accuracy",
#   "macro-f1",
#   "micro precision",
#   "macro recall (test_2020)",
#   "tim partial f1",
#   "auprc",
#   "swiss german test accuracy",
#   "validation (matched) f1",
#   "japanese test accuracy",
#   "ndcg_at_700",
#   "joint goal expected calibration error",
#   "best f1 thresh",
#   "bertscore_precision",
#   "best f1",
#   "f1 weighted",
#   "micro f1 (cardiffnlp/tweet_sentiment_multilingual/all)",
#   "weighted-f1",
#   "pearson_correlation",
#   "recall@5",
#   "f-measure",
#   "south levantine arabic test accuracy",
#   "test per on multilingual librispeech fr | trained",
#   "validation_accuracy",
#   "answer_exact_match_answer_extraction",
#   "naija test accuracy",
#   "recall_at_70",
#   "test (bokm\u00e5l) cer",
#   "bleu-4",
#   "micro f1 (cardiffnlp/tweet_topic_single)",
#   "precision_at_3",
#   "map_at_2",
#   "f1 (3-shot)",
#   "multilabel roc auc",
#   "cider",
#   "upos accuracy",
#   "accuracy (cardiffnlp/tweet_sentiment_multilingual/all)",
#   "epochs",
#   "mrr_at_3",
#   "roc auc",
#   "test wer (with lm)",
#   "300 samples, greedy decoding",
#   "gemini-correctness-score",
#   "test bertscore fanpage",
#   "val per on common voice fr 13.0 | trained",
#   "validation rouge-l sum",
#   "arc_challenge (0-shot)",
#   "test wer (+lm)",
#   "en_title_to_content_acc",
#   "precision_at_1",
#   "turkish test accuracy",
#   "ndcg_at_50",
#   "rougelsum",
#   "test bertscore",
#   "mrr_at_30",
#   "test accuracy",
#   "eval rougel",
#   "macro recall",
#   "accuracy (10-shot)",
#   "spider",
#   "ndcg_at_30",
#   "maltese test accuracy",
#   "truthfulqa:mc (0-shot)",
#   "test rogue-2",
#   "old east slavic test accuracy",
#   "estonian test accuracy",
#   "mrr_at_70",
#   "weighted precision",
#   "f1 (macro)",
#   "multilabel accuracy",
#   "tatsu-lab/alpaca_eval",
#   "macro f1 (cardiffnlp/tweet_sentiment_multilingual/all)",
#   "accuracy (tweet_eval/sentiment)",
#   "dot_spearman",
#   "chukchi test accuracy",
#   "croatian test accuracy",
#   "fid",
#   "chrf",
#   "latvian test accuracy",
#   "eval f1 score (squad metric)",
#   "cantonese test accuracy",
#   "pearsonr",
#   "rouge-l (question generation)",
#   "rougel",
#   "euclidean_recall",
#   "moverscore (question & answer generation)",
#   "marathi test accuracy",
#   "test wer (without lm)",
#   "mrr_at_10",
#   "wer_without_norm",
#   "micro f1 (tweet_eval/sentiment)",
#   "dot_pearson",
#   "eval wer",
#   "all samples, greedy decoding",
#   "map_at_1000",
#   "akuntsu test accuracy",
#   "qaalignedrecall-moverscore (question & answer generation (with gold answer))",
#   "precision micro",
#   "map_at_500",
#   "pearson's r (distress)",
#   "mc2",
#   "map_at_300",
#   "ndcg_at_300",
#   "eval_loss",
#   "sacrebleu chrf",
#   "recall_at_30",
#   "validation (mismatched) f1",
#   "top-1 accuracy",
#   "rouge-l (f-measure)",
#   "recall_at_300",
#   "hasans_f1",
#   "validation rouge-1",
#   "test mer",
#   "ndcg_at_10",
#   "qaalignedprecision-moverscore (question & answer generation (with gold answer))",
#   "bertscore recall",
#   "ter",
#   "de_content_to_title_acc",
#   "spearmanr",
#   "wer lm",
#   "precision",
#   "classical chinese test accuracy",
#   "pass@1 (java)",
#   "macro avg",
#   "eval f1",
#   "exact",
#   "micro f1 (tweet_eval/emoji)",
#   "indonesian test accuracy",
#   "mrr_at_5",
#   "f1neg",
#   "uyghur test accuracy",
#   "eval_hasans_exact",
#   "entity span recall (test_2020)",
#   "micro f1",
#   "precision_samples",
#   "macro recall (test_2021)",
#   "test f1 (callsign)",
#   "accuracy (tweet_eval/emotion)",
#   "winogrande (5-shot)",
#   "mrr_at_500",
#   "faroese test accuracy",
#   "precision_at_50",
#   "0",
#   "assyrian test accuracy",
#   "precision_micro",
#   "bertscore",
#   "winogrande",
#   "test f1 (macro)",
#   "qaalignedrecall-bertscore (question & answer generation)",
#   "meteor (question generation)",
#   "entity span f1 (test_2021)",
#   "mrr_at_7",
#   "test suite sql eval - execution accuracy",
#   "total_reward",
#   "pass@100 (t=0.8)",
#   "total",
#   "accuracy_score",
#   "accuracy 'bezeichnung'",
#   "test rouge-l sum",
#   "accuracy (0 shot)",
#   "macro f1 (test_2020)",
#   "test (bokm\u00e5l) wer",
#   "rouge-1-f",
#   "map_at_70",
#   "romanian test accuracy",
#   "accuracy (tweet_eval/irony)",
#   "manhattan_pearson",
#   "micro f1 (tweet_eval/emotion)",
#   "english test accuracy",
#   "mixture accuracy",
#   "accuracy (0-shot)",
#   "ndcg_at_5",
#   "czech test accuracy",
#   "cer-char",
#   "code_eval_outputs",
#   "mrr_at_300",
#   "em",
#   "dev wer (+lm)",
#   "recall_at_2",
#   "micro f1 (tweet_eval/offensive)",
#   "test accuracy on coscan speech",
#   "ldm3d-sr-b psnr",
#   "precision_weighted",
#   "dvitel/codebleu",
#   "ndcg_at_1000",
#   "arc",
#   "test pearson correlation coefficient",
#   "map_at_10",
#   "map_at_20",
#   "test suite sql eval - exact matching accuracy",
#   "euclidean_pearson",
#   "mean_reciprocal_rank",
#   "rouge2_acc",
#   "de_title_to_content_acc",
#   "f1 (seqeval)",
#   "pass@10 (java)",
#   "unknown",
#   "meteor (question & answer generation)",
#   "test qwk",
#   "question_to_text_acc",
#   "pass@1 (t=0.1)",
#   "old church slavonic test accuracy",
#   "cos_sim_pearson",
#   "test rougel",
#   "mer",
#   "train cer",
#   "map_at_200",
#   "qaalignedprecision-moverscore (question & answer generation)",
#   "warlpiri test accuracy",
#   "test ser",
#   "serbian test accuracy",
#   "test per",
#   "qaalignedf1score-moverscore (question & answer generation)",
#   "f1_micro",
#   "eval cer",
#   "mt-bench",
#   "map_at_50",
#   "hellaswag (10-shot)",
#   "test coraa wer",
#   "validation rouge-2",
#   "precision@5",
#   "ndcg_at_20",
#   "euclidean_precision",
#   "kv partial f1",
#   "test rouge2 fanpage",
#   "validation f1 (micro) on coscan speech",
#   "macro precision (test_2020)",
#   "test recall",
#   "macro precision",
#   "accuracy (tweet_eval/hate)",
#   "test rouge1 ilpost",
#   "arc easy",
#   "validation rouge-l",
#   "test (nynorsk) cer",
#   "recall@10",
#   "validation rogue-l-sum",
#   "loss",
#   "albanian test accuracy",
#   "qaalignedf1score-moverscore (question & answer generation) [gold answer]",
#   "1",
#   "rouge-l (question answering)",
#   "eval sari",
#   "validation rogue-2",
#   "validation wer with 5-gram lm",
#   "openbookqa",
#   "precision_at_20",
#   "precision 'thema' (macro)",
#   "noans_f1",
#   "avg. f1",
#   "accuracy (25-shot)",
#   "hindi test accuracy",
#   "5-way 5~10-shot",
#   "eval_hasans_f1",
#   "lithuanian test accuracy",
#   "wil",
#   "manhattan_ap",
#   "moverscore (answer extraction)",
#   "precision_at_70",
#   "mean f1 measure of muc, bcubed, and ceafe",
#   "eval_f1",
#   "dot_precision",
#   "sanskrit test accuracy",
#   "test rougelsum",
#   "zero-shot transfer",
#   "bertscore_f1",
#   "precision_at_200",
#   "accuracy (5 shot)",
#   "normalized accuracy",
#   "recall_at_50",
#   "cer lm",
#   "khunsari test accuracy",
#   "f1 'bezeichnung' (macro)",
#   "yoruba test accuracy",
#   "nayini test accuracy",
#   "precision (macro)",
#   "validation rogue-l",
#   "answer_f1_score__answer_extraction",
#   "test cer  (+lm)",
#   "test rouge1",
#   "test wer (with language model)",
#   "qaalignedrecall-bertscore (question & answer generation (with gold answer)) [gold answer]",
#   "validation (mismatched) accuracy",
#   "macro f1 (tweet_eval/hate)",
#   "polish test accuracy",
#   "en_question_to_text_acc",
#   "test jaccard error rate",
#   "rougel_max",
#   "exact match",
#   "bertscoref1",
#   "slot error rate",
#   "accuracy_manhattan",
#   "recall (macro)",
#   "rouge2_max",
#   "validation rogue-lsum",
#   "bleu4 (question & answer generation)",
#   "ap",
#   "catalan test accuracy",
#   "rouge-1-r",
#   "upper sorbian test accuracy",
#   "average f1",
#   "unlabelled attachment score",
#   "mrr_at_700",
#   "steps per second",
#   "gsm8k",
#   "confusion_matrix",
#   "accuracy@10",
#   "f1 score",
#   "wer (reference column: transcription)",
#   "entity span f1 (test_2020)",
#   "avg. test rouge1",
#   "n/a",
#   "eval exactmatch score (squad metric)",
#   "qaalignedf1score-bertscore (question & answer generation) [gold answer]",
#   "test per (w/o stress)",
#   "validation precision",
#   "macro f1 (tweet_eval/emotion)",
#   "n.a.",
#   "vietnamese test accuracy",
#   "qaalignedrecall-moverscore (question & answer generation (with gold answer)) [gold answer]",
#   "accuracy_euclidean",
#   "mse loss",
#   "byte_perplexity",
#   "bertscore (f1)",
#   "average",
#   "bertscore (question & answer generation)",
#   "rouge-l",
#   "hebrew test accuracy",
#   "micro recall",
#   "komi permyak test accuracy",
#   "cer-rome",
#   "recall_at_700",
#   "prompt_level_loose_acc",
#   "russian test accuracy",
#   "euclidean_accuracy",
#   "bertscore precision",
#   "qaalignedrecall-bertscore (question & answer generation) [gold answer]",
#   "+++",
#   "mse",
#   "gsm8k (5-shot)",
#   "bleu_diff",
#   "ndcg_at_200",
#   "test exact match",
#   "recall_at_7",
#   "ndcg_at_100",
#   "validation recall",
#   "total time in seconds",
#   "de_question_to_text_acc",
#   "combined_score",
#   "precision_at_700",
#   "macro_precision",
#   "latency in seconds",
#   "toxicity",
#   "precision_at_30",
#   "recall macro",
#   "english to chinese",
#   "rouge-l-r",
#   "latin test accuracy",
#   "f1 micro",
#   "v_measure",
#   "human_normalized_total_reward",
#   "galician test accuracy",
#   "meteor (question answering)",
#   "common voice irish invalidated 281 utterances (without lm)",
#   "entity span precision (test_2020)",
#   "dummy metric",
#   "5 way 1~2 shot",
#   "mbya guarani test accuracy",
#   "accuracy 'thema'",
#   "macro f1 (cardiffnlp/tweet_topic_single)",
#   "code_eval",
#   "results partial f1",
#   "qaalignedrecall-moverscore (question & answer generation) [gold answer]",
#   "accuracy@5",
#   "noans_total",
#   "dev cer (+lm)",
#   "afrikaans test accuracy",
#   "dev cer (without lm)",
#   "cer",
#   "boolq",
#   "win rate",
#   "map_at_100",
#   "qaalignedf1score-bertscore (question & answer generation (with gold answer)) [gold answer]",
#   "winogrande (0-shot)",
#   "squad f1",
#   "precision (test_2020)",
#   "macro f1 (cardiffnlp/tweet_topic_multi)",
#   "moverscore (question generation)",
#   "test per on common voice fr 13.0 | trained",
#   "iqm_expert_normalized_total_reward",
#   "precision_at_300",
#   "munduruku test accuracy",
#   "accuracy (tweet_eval/offensive)",
#   "global strict f1",
#   "precision (entity span)",
#   "test f1 (command)",
#   "empos",
#   "bleu",
#   "north sami test accuracy",
#   "recall_micro",
#   "common voice irish invalidated 281 utterances (with lm)",
#   "slovak test accuracy",
#   "precision_at_2",
#   "macro f1 (tweet_eval/irony)",
#   "mrr_at_50",
#   "samples per second",
#   "french test accuracy",
#   "map_at_30",
#   "pass@100",
#   "slot f1",
#   "recall (entity span)",
#   "micro_recall",
#   "macro precision (test_2021)",
#   "f1 macro",
#   "glue",
#   "f1@m (absent)",
#   "test wer (no lm)",
#   "alpacaeval",
#   "cos_sim_precision",
#   "avg. test bertscore",
#   "xpos accuracy",
#   "makurap test accuracy",
#   "pass@10 (javascript)",
#   "ndcg_at_3",
#   "tupinamba test accuracy",
#   "test f1",
#   "exact_match",
#   "recall_weighted",
#   "precision_at_7",
#   "dot_accuracy",
#   "moverscore (question answering)",
#   "en_text_to_question_acc",
#   "recall_at_1",
#   "macro_f1",
#   "wer",
#   "ldm3d-sr-b is",
#   "ai2 reasoning challenge (25-shot)",
#   "test wer with lm",
#   "f1 (entity span)",
#   "validation wer",
#   "hasans f1",
#   "weighted_recall",
#   "dev wer (with lm)",
#   "f1@o (absent)",
#   "arc (25-shot)",
#   "test wil",
#   "kaapor test accuracy",
#   "ndcg_at_70",
#   "pearson correlation - stsb_multi_mt fr",
#   "noans_exact",
#   "cos_sim_recall",
#   "ldm3d-sr-b depth mare",
#   "accuracy (cardiffnlp/tweet_topic_single)",
#   "moksha test accuracy",
#   "test rouge-1",
#   "sacrebleu",
#   "test macro f1",
#   "joint goal accuracy",
#   "f1",
#   "finnish test accuracy",
#   "false accuracy",
#   "gothic test accuracy",
#   "qaalignedrecall-bertscore (question & answer generation (with gold answer))",
#   "ancient greek test accuracy",
#   "test f1 (value)",
#   "joint validation accuracy",
#   "bleurt_max",
#   "test headline-headline consistency accuracy",
#   "2",
#   "manx test accuracy",
#   "test rouge-l",
#   "eval_exact",
#   "roc_auc",
#   "validation accuracy on coscan speech",
#   "qaalignedprecision-bertscore (question & answer generation (with gold answer)) [gold answer]",
#   "micro-f1 score",
#   "dot_f1",
#   "dev wer (without lm)",
#   "arabic test accuracy",
#   "validation (matched) accuracy",
#   "kangri test accuracy",
#   "test cer (without lm)",
#   "mrr_at_2",
#   "ip partial f1",
#   "judge_acc",
#   "portuguese test accuracy",
#   "livvi test accuracy",
#   "map_at_1",
#   "test precision (macro)",
#   "test wer using lm",
#   "f1_weighted",
#   "accuracy_cosinus",
#   "precision_at_100",
#   "metric",
#   "rouge1_acc",
#   "meteor",
#   "pass@10",
#   "squad_v2",
#   "dev wer",
#   "dialog acts f1",
#   "macro f1 (test_2021)",
#   "en_content_to_title_acc",
#   "mrr_at_200",
#   "korean test accuracy",
#   "test precision",
#   "qaalignedprecision-moverscore (question & answer generation) [gold answer]",
#   "test/f1",
#   "kazakh test accuracy",
#   "cos_sim_spearman",
#   "precision 'bezeichnung' (macro)",
#   "meteor (answer extraction)",
#   "f1@o (present)",
#   "cosine_score_all-minilm-l6-v2",
#   "tim strict f1",
#   "german test accuracy",
#   "recall (test_2021)",
#   "bulgarian test accuracy",
#   "expert_normalized_total_reward",
#   "max_f1",
#   "manhattan_precision",
#   "recall_at_500",
#   "humanities",
#   "language model loss",
#   "true accuracy",
#   "kv strict f1",
#   "eval_noans_exact",
#   "test wip",
#   "ip strict f1",
#   "rouge-2",
#   "recall_at_5",
#   "test cer with lm",
#   "basque test accuracy",
#   "answer_exact_match_question_answering",
#   "edit-smiliarity",
#   "codebleu",
#   "chr-f",
#   "\u226490%ile",
#   "recall micro",
#   "recall_samples",
#   "eval_noans_f1",
#   "bleurt_acc",
#   "macro f1 (tweet_eval/emoji)",
#   "dot_ap",
#   "karelian test accuracy",
#   "tagalog test accuracy",
#   "avg. test rougel",
#   "belarusian test accuracy",
#   "10-way 1~2-shot",
#   "eval rougelsum",
#   "low saxon test accuracy",
#   "bleurt_diff",
#   "gen_len",
#   "f1 (macro avg.)",
#   "bleu4 (question answering)",
#   "f1 (test_2021)",
#   "bleu score",
#   "spearman corr",
#   "test recall (macro)",
#   "test sari",
#   "qaalignedprecision-bertscore (question & answer generation (with gold answer))",
#   "dev cer (with lm)",
#   "em (3-shot)",
#   "recall_at_100",
#   "drop (3-shot)",
#   "recall_at_1000",
#   "eval exact",
#   "dialog acts accuracy",
#   "pearson correlation",
#   "test cer  with lm",
#   "validation cer with 5-gram lm",
#   "eval rouge-2",
#   "piqa",
#   "irish test accuracy",
#   "rouge2",
#   "single-line infilling pass@1",
#   "wer (reference column: raw_transcription)",
#   "weighted f1",
#   "pass@10 (t=0.8)",
#   "thai test accuracy",
#   "validation accuracy",
#   "recall_at_20",
#   "western armenian test accuracy",
#   "macro-averaged f1",
#   "perplexity",
#   "komi zyrian test accuracy",
#   "recall 'thema' (macro)",
#   "bleu4 (question generation)",
#   "pass@1 (t=0.2)",
#   "avg. test rouge2",
#   "akkadian test accuracy",
#   "accuracy",
#   "mrr",
#   "weighted recall",
#   "test headline-article consistency accuracy",
#   "test rogue-l",
#   "qaalignedprecision-moverscore (question & answer generation (with gold answer)) [gold answer]",
#   "test (nynorsk) wer",
#   "euclidean_spearman",
#   "euclidean_ap",
#   "f1 (test_2020)",
#   "pass@1 (python)",
#   "precision_at_5",
#   "micro_precision",
#   "lambda",
#   "persian test accuracy",
#   "validation rogue-1",
#   "best exact thresh",
#   "self-reported",
#   "val per",
#   "cos_sim_ap",
#   "manhattan_accuracy",
#   "test rougel ilpost",
#   "precision (test_2021)",
#   "squad em",
#   "old french test accuracy",
#   "wiki_split",
#   "unproven accuracy",
#   "scottish gaelic test accuracy",
#   "micro f1 (tweet_eval/irony)",
#   "validation loss",
#   "stem",
#   "italian test accuracy",
#   "clipsim",
#   "rougel_acc",
#   "bleu_max",
#   "map_at_3",
#   "is",
#   "erzya test accuracy",
#   "rmse [m]",
#   "rouge-lsum",
#   "spanish test accuracy",
#   "weighted_precision",
#   "bertscore (answer extraction)",
#   "test wer",
#   "qaalignedprecision-bertscore (question & answer generation) [gold answer]",
#   "normalized cer",
#   "macro f1 (tweet_eval/offensive)",
#   "wer (without normalization)",
#   "alpacaeval 1.0",
#   "f1-macro",
#   "social science",
#   "rouge1_diff",
#   "mrr_at_1000",
#   "span-based f1",
#   "hasans_total",
#   "max_accuracy",
#   "micro-f1",
#   "best exact",
#   "pearson corr",
#   "f1@m",
#   "mrr_at_1",
#   "rouge2_diff",
#   "rouge1",
#   "pass@1 (javascript)",
#   "mmlu",
#   "seqeval",
#   "mmlu (5-shot)",
#   "test cer",
#   "pearson's r (empathy)",
#   "micro_f1",
#   "chinese test accuracy",
#   "weighted avg",
#   "answer_f1_score__question_answering",
#   "guajajara test accuracy",
#   "wolof test accuracy",
#   "dev cer",
#   "precision weighted",
#   "cos_sim_accuracy",
#   "f1@m (present)",
#   "pass@10 (python)",
#   "rougel_diff",
#   "recall weighted",
#   "test rogue-lsum",
#   "validation rogue-1.",
#   "re+ macro f1",
#   "rouge-l-p",
#   "bertscore (question answering)",
#   "iqm_human_normalized_total_reward",
#   "pass@1",
#   "bertscore (question generation)",
#   "eval rouge-1",
#   "tamil test accuracy",
#   "manhattan_spearman",
#   "bleu4 (answer extraction)",
#   "map_at_700",
#   "weighted_f1",
#   "mean_reward",
#   "test f1 (micro) on coscan speech",
#   "toxicity rito",
#   "f1pos",
#   "bhojpuri test accuracy",
#   "mae",
#   "10-way 5~10-shot",
#   "hamming score",
#   "test cer (no lm)",
#   "hasans_exact",
#   "recall_at_200",
#   "classification_report",
#   "precision macro",
#   "precision_at_1000",
#   "manhattan_f1",
#   "mrr_at_20",
#   "accuracy (cardiffnlp/tweet_topic_multi)",
#   "test rouge2 ilpost",
#   "ndcg_at_1",
#   "icelandic test accuracy",
#   "ndcg_at_2",
#   "validation perplexity",
#   "arc challenge",
#   "ukrainian test accuracy",
#   "rouge-1-p",
#   "euclidean_f1",
#   "qaalignedf1score-moverscore (question & answer generation (with gold answer)) [gold answer]",
#   "telugu test accuracy",
#   "test cer (with lm)",
#   "bertscore_recall",
#   "alpacaeval 2.0",
#   "bertscore f1",
#   "other",
#   "ldm3d-sr-b fid",
#   "recall 'bezeichnung' (macro)",
#   "validation cer",
#   "rouge1_max",
#   "labelled attachment score",
#   "absrel",
#   "pass@1 (t=0.01)",
#   "perplexity (wip)",
#   "test cer (w/o stress)",
#   "f1 'thema' (macro)",
#   "accuracy top2",
#   "hungarian test accuracy",
#   "rouge-l (question & answer generation)",
#   "percision",
#   "hasans exact",
#   "swedish test accuracy",
#   "welsh test accuracy",
#   "test rouge1 fanpage",
#   "macro f1",
#   "qaalignedf1score-bertscore (question & answer generation (with gold answer))",
#   "test rougel fanpage",
#   "accuracy (tweet_eval/emoji)",
#   "test rouge-2",
#   "recall (test_2020)",
#   "rouge-1",
#   "exact-match",
#   "wip",
#   "rouge-l (answer extraction)",
#   "danish test accuracy",
#   "skolt sami test accuracy",
#   "squad",
#   "qaalignedrecall-moverscore (question & answer generation)",
#   "recall",
#   "training loss",
#   "validationloss",
#   "mrr_at_100",
#   "truthfulqa",
#   "bambara test accuracy",
#   "qaalignedf1score-bertscore (question & answer generation)",
#   "ndcg_at_500",
#   "test cer (+lm)",
#   "single-line infilling pass@10",
#   "precision_at_500",
#   "f1-score",
#   "ldm3d-sr-b ssim",
#   "test cer using lm",
#   "mauve",
#   "roc-auc",
#   "recall_at_10",
#   "norwegian test accuracy",
#   "apurina test accuracy",
#   "manhattan_recall",
#   "pearson",
#   "spice",
#   "slovenian test accuracy",
#   "rouge",
#   "max_ap",
#   "old turkish test accuracy",
#   "recall_at_3",
#   "wer (beam 5)",
#   "precision_at_10",
#   "micro f1 (tweet_eval/hate)",
#   "rouge-l-f",
#   "acc",
#   "bleu_acc",
#   "urdu test accuracy",
#   "map",
#   "map_at_7",
#   "test wer on common voice 7",
#   "micro f1 (cardiffnlp/tweet_topic_multi)",
#   "mare",
#   "wer (greedy)",
#   "text_to_question_acc",
#   "clip",
#   "macro f1 (tweet_eval/sentiment)",
#   "test rogue-1",
#   "de_text_to_question_acc",
#   "ndcg_at_7",
#   "matthews_correlation",
#   "validation macro f1",
#   "hasans total",
#   "qaalignedf1score-moverscore (question & answer generation (with gold answer))",
#   "macro_recall",
#   "kiche test accuracy",
#   "test wer without lm",
#   "entity span recall (test_2021)",
#   "test bertscore ilpost",
#   "grade school math 8k (5-shot)",
#   "dot_recall",
#   "hellaswag",
#   "dutch test accuracy",
#   "top-3-accuracy",
#   "test spearmanr",
#   "qaalignedprecision-bertscore (question & answer generation)",
#   "validation f1",
#   "f1_macro",
#   "hellaswag (0-shot)",
#   "breton test accuracy",
#   "truthfulqa (0-shot)",
#   "buryat test accuracy",
#   "armenian test accuracy",
#   "top-5 accuracy",
#   "greek test accuracy"
# ]

# x = list()
# for i in ii:
#     x.append(parse_metric(i))

# import json
# print(json.dumps(list(set(x)), indent=2))
# print(len(set(x)))

#ii = [
  #"common voice sah",
  #"mozilla-foundation/common_voice_11_0 nl",
  #"common voice et",
  #"common voice zh-hk",
  #"mozilla-foundation/common_voice_11_0 es",
  #"mozilla-foundation/common_voice_16_0 tr",
  #"mozilla-foundation/common_voice_11_0 bn",
  #"invoice",
  #"mozilla-foundation/common_voice_16_0 fa",
  #"mozilla-foundation/common_voice_11_0 fy-nl",
  #"common voice hi",
  #"mozilla-foundation/common_voice_11_0 id",
  #"mozilla-foundation/common_voice_16_0 ko",
  #"common voice 13.0 fa",
  #"common voice gl",
  #"common voice fi",
  #"common_voice_6_1",
  #"mozilla-foundation/common_voice_8_0 es",
  #"mozilla-foundation/common_voice_11_0 ka",
  #"mozilla-foundation/common_voice_13_0 cs",
  #"mozilla-foundation/common_voice_11_0 fi",
  #"mozilla-foundation/common_voice_13_0 th",
  #"common voice fr",
  #"common voice dv",
  #"mozilla-foundation/common_voice_16_0 - vi",
  #"common voice rm-vallader",
  #"language-and-voice-lab/althingi_asr",
  #"common voice th",
  #"mozilla-foundation/common_voice_10_0 uk",
  #"mozilla-foundation/common_voice_11_0 pa-in",
  #"mozilla-foundation/common_voice_11_0 hi",
  #"common voice cy",
  #"common_voice_11_0",
  #"mozilla-foundation/common_voice_11_0 sl",
  #"mozilla-foundation/common_voice_11_0 sv-se",
  #"haseong8012/korean-child-command-voice_train-0-10000_smaplingrate-16000",
  #"mozilla-foundation/common_voice_11_0",
  #"mozilla-foundation/common_voice_11_0 el",
  #"mozilla-foundation/common_voice_9_0 id",
  #"elite35p-server/elitevoiceproject twitter",
  #"common voice ky",
  #"mozilla-foundation/common_voice_11_0 hu",
  #"evanarlian/common_voice_11_0_id_filtered",
  #"mozilla-foundation/common_voice_11_0 sr",
  #"mozilla-foundation/common_voice_13_0 dv",
  #"common_voice_10_0",
  #"mozilla-foundation/common_voice_11_0 zh-cn",
  #"common voice ru",
  #"common voice ia",
  #"common voice ja",
  #"mozilla-foundation/common_voice_6_1 - ta",
  #"alxfng/noisycommonvoice",
  #"mozilla-foundation/common_voice_11_0 uk",
  #"mozilla-foundation/common_voice_11_0 da",
  #"mozilla-foundation/common_voice_11_0 ur",
  #"mozilla-foundation/common_voice_11_0 mt",
  #"common voice (urdu)",
  #"mozilla-foundation/common_voice_16_1",
  #"mozilla-foundation/common_voice_11_albanian",
  #"common voice ca",
  #"common voice mt",
  #"common voice hu",
  #"mozilla-foundation/common_voice_13_0 - sw",
  #"common_voice_16_1",
  #"mozilla-foundation/common_voice_16_0 pt",
  #"mozilla-foundation/common_voice_16_0 hi",
  #"mozilla-foundation/common_voice_11_0, google/fleurs, openslr, collectivat/tv3_parla, projecte-aina/parlament_parla",
  #"mozilla-foundation/common_voice_16_0 yue",
  #"mozilla-foundation/common_voice_16_0 fr",
  #"common voice eo",
  #"mozilla-foundation/common_voice_13_0 es",
  #"mozilla-foundation/common_voice_16_0 ml",
  #"artyomboyko/common_voice_15_0_ru",
  #"mozilla-foundation/common_voice_11_0 vi",
  #"mozilla-foundation/common_voice_11_0 en",
  #"common voice lv",
  #"common voice nl",
  #"common_voice - it",
  #"mozilla-foundation/common_voice_13_0 - de",
  #"common voice corpus 6.1 (clean)",
  #"common voice cs",
  #"common_voice_8_0",
  #"mozilla-foundation/common_voice_11_0 cs",
  #"mozilla-foundation/common_voice_11_0 ba",
  #"common voice el",
  #"mozilla-foundation/common_voice_13_0 - tr",
  #"mozilla-foundation/common_voice_16_0 id",
  #"mozilla-foundation/common_voice_11_0 zh-hk",
  #"common voice ta",
  #"common voice vox populi swedish",
  #"mozilla-foundation/common_voice_13_0 gl",
  #"common voice rw",
  #"common voice 13",
  #"mozilla-foundation/common_voice_7_0",
  #"common voice as",
  #"mozilla-foundation/common_voice_11_0 pt",
  #"haseong8012/korean-child-command-voice_train-0-10000",
  #"common voice vi",
  #"language-and-voice-lab/samromur_children",
  #"mozilla-foundation/common_voice_11_0 de",
  #"common voice hsb",
  #"commonvoice (clean)",
  #"common voice tt",
  #"mozilla-foundation/common_voice_11_0 it",
  #"mozilla-foundation/common_voice_13_0",
  #"mozilla-foundation/common_voice_11_0 - ur",
  #"common voice de",
  #"mozilla-foundation/common_voice_16_0 eu",
  #"language-and-voice-lab/samromur_asr",
  #"mozilla-foundation/common_voice_11_0 gl",
  #"common_voice - vi",
  #"mozilla-foundation/common_voice_11_0 zh-tw",
  #"common_voice_12_0",
  #"mozilla-foundation/common_voice_11_0 rw",
  #"common voice pl",
  #"mozilla-foundation/common_voice_11_0 sw",
  #"common voice fa",
  #"mozilla-foundation/common_voice_11_0 ar",
  #"vlads159/common_voice_16_1_romanian_speech_synthesis",
  #"mozilla-foundation/common_voice_11_0 nan-tw",
  #"common voice fy-nl",
  #"mozilla-foundation/common_voice_11_0 be",
  #"mozilla-foundation/common_voice_9_0",
  #"mozilla-foundation/common_voice_16_1 pt",
  #"common voice lt",
  #"mnazari/urmi-assyrian-voice",
  #"mozilla-foundation/common_voice_11_0 uz",
  #"mozilla-foundation/common_voice_16_0 - hi",
  #"common voice ka",
  #"mozilla-foundation/common_voice_15_0",
  #"mozilla-foundation/common_voice_8_0 ca",
  #"mozilla-foundation/common_voice_11_0 az",
  #"mozilla-foundation/common_voice_11_0 kk",
  #"mozilla-foundation/common_voice_11_0 mr",
  #"common voice pt",
  #"mozilla-foundation/common_voice_11_0 hy-am",
  #"language-and-voice-lab/malromur_asr",
  #"mozilla-foundation/common_voice_11_0 ml",
  #"mozilla-foundation/common_voice_11_0 tr",
  #"mozilla-foundation/common_voice_11_0 as",
  #"common voice rm-sursilv",
  #"common voice (bengali)",
  #"mozilla-foundation/common_voice_16_0 ta",
  #"common voice 6",
  #"common voice ???",
  #"mozilla-foundation/common_voice_11_0 fa",
  #"mozilla-foundation/common_voice_11_0 ta",
  #"mozilla-foundation/common_voice_11_0 ca",
  #"common voice en",
  #"mozilla-foundation/common_voice_16_0 - id",
  #"common voice ar",
  #"mozilla-foundation/common_voice_13_0 bg",
  #"common_voice_13_0",
  #"mozilla-foundation/common_voice_11_0 ja",
  #"common voice japanese",
  #"mozilla-foundation/common_voice_11_0 ro",
  #"mozilla-foundation/common_voice_11_0 eu",
  #"mozilla-foundation/common_voice_11_0 lv",
  #"common voice cnh",
  #"common_voice_14_0",
  #"common voice uk",
  #"mozilla-foundation/common_voice_11_0 ha",
  #"common voice br",
  #"mozilla-foundation/common_voice_11_0 or",
  #"common_voice_15",
  #"invoices",
  #"mozilla-foundation/common_voice_13_0 eu",
  #"vlads159/common_voice_romanian_speech_synthesis",
  #"common voice cv",
  #"common voice ro",
  #"common_voice_7_0",
  #"mozilla-foundation/common_voice_16_0 - ps",
  #"mozilla-foundation/common_voice_11_0 ru",
  #"common_voice rw",
  #"mozilla-foundation/common_voice_11_0,facebook/voxpopuli,google/fleurs de,de,de_de",
  #"common voice pa-in",
  #"mozilla-foundation/common_voice_16_0 - tr",
  #"common voice es",
  #"common_voice_15_0",
  #"mozilla-foundation/common_voice_15_0 - hi",
  #"invoice_layoutlmv3",
  #"mozilla-foundation/common_voice_16_0 - ml",
  #"mozilla-foundation/common_voice_11_0 bg",
  #"common voice id",
  #"common voice or",
  #"common voice 13 - whisper_small on asr_ja ",
  #"mozilla-foundation/common_voice_11_0 tt",
  #"mozilla-foundation/common_voice_11_0 - tr",
  #"mozilla-foundation/common_voice_11_0 kab",
  #"common voice 6.1",
  #"vivos + commonvoice",
  #"mozilla-foundation/common_voice_6_1",
  #"common voice lg",
  #"common voice 10.0",
  #"mozilla-foundation/common_voice_11_0 np-np",
  #"mozilla-foundation/common_voice_9",
  #"mozilla-foundation/common_voice_10_0",
  #"common voice mn",
  #"ccs10 + mozilla-foundation/common_voice_7_0",
  #"mozilla-foundation/common_voice_11_0 th",
  #"mozilla-foundation/common_voice_16_albanian",
  #"common-voice-vietnamese",
  #"common voice zh-cn",
  #"common voice sl",
  #"mozilla-foundation/common_voice_3_0",
  #"common_voice lg",
  #"common voice eu",
  #"common voice estonian",
  #"common_voice - tr",
  #"mozilla-foundation/common_voice_16_0 bn",
  #"mozilla-foundation/common_voice_11_0,google/fleurs sr,sr_rs",
  #"mozilla-foundation/common_voice_11_0 ne-np",
  #"mozilla-foundation/common_voice_11_0 cy",
  #"common voice luganda",
  #"mozilla-foundation/common_voice_11_0 fr",
  #"mozilla-foundation/common_voice_16_0 ar",
  #"common_voice",
  #"tamilcustomvoice",
  #"common_voice es",
  #"mozilla-foundation/common_voice_16_0 - af",
  #"mozilla common voices - 16.0 - portuguese",
  #"mozilla-foundation/common_voice_13_0 ca",
  #"mozilla-foundation/common_voice_14_0",
  #"mozilla-foundation/common_voice_16_1 lv",
  #"mozilla-foundation/common_voice_11_0,google/fleurs el,el_gr",
  #"common voice made up words",
  #"mozilla-foundation/common_voice_11_0 br",
  #"common_voice_1_0",
  #"common voice zh-tw",
  #"mozilla-foundation/common_voice_11",
  #"common_voice_16_0",
  #"mozilla-foundation/common_voice_8_0",
  #"common voice sv-se",
  #"mozilla-foundation/common_voice_12_0",
  #"elite35p-server/elitevoiceproject youtube",
  #"common voice tr",
  #"common_voice - hi",
  #"mozilla-foundation/common_voice_16_0",
  #"mozilla-foundation/common_voice_11_0 sk",
  #"common voice ga-ie",
  #"common voice pa",
  #"common voice it",
  #"mozilla commonvoice (vietnamese) v16.1"
#]
#x = [parse_dataset(i, 'test')[0] for i in ii]


#import json
#a = json.dumps(list(set(x)), indent=2)
#print(a)
#print(len(set(x)))
#print(len(set(ii)))
