import json
import argparse
import os
from pathlib import Path
import hashlib
import shutil

# default treebank for languages
default_treebanks = {
  "af": "afribooms",
  "grc": "proiel",
  "ar": "padt",
  "hy": "armtdp",
  "eu": "bdt",
  "bg": "btb",
  "bxr": "bdt",
  "ca": "ancora",
  "zh-hant": "gsd",
  "hr": "set",
  "cs": "pdt",
  "da": "ddt",
  "nl": "alpino",
  "en": "ewt",
  "et": "edt",
  "fi": "tdt",
  "fr": "gsd",
  "gl": "ctg",
  "de": "gsd",
  "got": "proiel",
  "el": "gdt",
  "he": "htb",
  "hi": "hdtb",
  "hu": "szeged",
  "id": "gsd",
  "ga": "idt",
  "it": "isdt",
  "ja": "gsd",
  "kk": "ktb",
  "ko": "kaist",
  "kmr": "mg",
  "la": "ittb",
  "lv": "lvtb",
  "sme": "giella",
  "cu": "proiel",
  "fro": "srcmf",
  "fa": "seraji",
  "pl": "lfg",
  "pt": "bosque",
  "ro": "rrt",
  "ru": "syntagrus",
  "sr": "set",
  "sk": "snk",
  "sl": "ssj",
  "es": "ancora",
  "sv": "talbanken",
  "tr": "imst",
  "uk": "iu",
  "hsb": "ufal",
  "ur": "udtb",
  "ug": "udt",
  "vi": "vtb",
  "lt": "hse",
  "wo": "wtb",
  "nb": "bokmaal",
  "mt": "mudt",
  "swl": "sslc",
  "cop": "scriptorium",
  "be": "hse",
  "zh-hans": "gsdsimp",
  "lzh": "kyoto",
  "gd": "arcosg",
  "olo": "kkpp",
  "ta": "ttb",
  "te": "mtg",
  "orv": "torot",
  "nn": "nynorsk",
  "mr": "ufal"
}

# add full language name to language code and add alias in resources
lcode2lang = {
    "af": "Afrikaans",
    "grc": "Ancient_Greek",
    "ar": "Arabic",
    "hy": "Armenian",
    "eu": "Basque",
    "be": "Belarusian",
    "br": "Breton",
    "bg": "Bulgarian",
    "bxr": "Buryat",
    "ca": "Catalan",
    "zh-hant": "Traditional_Chinese",
    "lzh": "Classical_Chinese",
    "cop": "Coptic",
    "hr": "Croatian",
    "cs": "Czech",
    "da": "Danish",
    "nl": "Dutch",
    "en": "English",
    "et": "Estonian",
    "fo": "Faroese",
    "fi": "Finnish",
    "fr": "French",
    "gl": "Galician",
    "de": "German",
    "got": "Gothic",
    "el": "Greek",
    "he": "Hebrew",
    "hi": "Hindi",
    "hu": "Hungarian",
    "id": "Indonesian",
    "ga": "Irish",
    "it": "Italian",
    "ja": "Japanese",
    "kk": "Kazakh",
    "ko": "Korean",
    "kmr": "Kurmanji",
    "lt": "Lithuanian",
    "olo": "Livvi",
    "la": "Latin",
    "lv": "Latvian",
    "mt": "Maltese",
    "mr": "Marathi",
    "pcm": "Naija",
    "sme": "North_Sami",
    "nb": "Norwegian_Bokmaal",
    "nn": "Norwegian_Nynorsk",
    "cu": "Old_Church_Slavonic",
    "fro": "Old_French",
    "orv": "Old_Russian",
    "fa": "Persian",
    "pl": "Polish",
    "pt": "Portuguese",
    "ro": "Romanian",
    "ru": "Russian",
    "gd": "Scottish_Gaelic",
    "sr": "Serbian",
    "zh-hans": "Simplified_Chinese",
    "sk": "Slovak",
    "sl": "Slovenian",
    "es": "Spanish",
    "sv": "Swedish",
    "swl": "Swedish_Sign_Language",
    "ta": "Tamil",
    "te": "Telugu",
    "th": "Thai",
    "tr": "Turkish",
    "uk": "Ukrainian",
    "hsb": "Upper_Sorbian",
    "ur": "Urdu",
    "ug": "Uyghur",
    "vi": "Vietnamese",
    "wo": "Wolof"
}

# default models for languages
default_models = {
  "ar": "ar_padt.asafaya",
  "hy": "armtdp",
  "eu": "bdt",
  "bg": "bg_btb.DeepPavlov",
  "bxr": "bdt",
  "ca": "ancora",
  "zh-hant": "gsd",
  "hr": "set",
  "cs": "cs_PDT.DeepPavlov",
  "da": "ddt",
  "nl": "nl_alpino.wietsedv",
  "en": "en_ewt.electra-base",
  "et": "et_edt_ewt.TurkuNLP",
  "fi": "fi_tdt.TurkuNLP",
  "fr": "fr_sequoia.camembert-large",
  "gl": "ctg",
  "de": "gsd",
  "got": "proiel",
  "el": "gdt",
  "he": "htb",
  "hi": "hdtb",
  "hu": "szeged",
  "id": "gsd",
  "ga": "idt",
  "it": "it_isdt.dbmdz-xxl",
  "ja": "gsd",
  "kk": "ktb",
  "ko": "kaist",
  "kmr": "mg",
  "la": "ittb",
  "lt": "lt_alksnis.wietsedve",
  "lv": "lvtb",
  "sme": "giella",
  "cu": "proiel",
  "fro": "srcmf",
  "fa": "seraji",
  "pl": "pl_pdb_pud.dkleczek",
  "pt": "bosque",
  "ro": "rrt",
  "ru": "ru_syntagrus.DeepPavlov",
  "sr": "set",
  "sk": "sk_snk.KB",
  "sl": "ssj",
  "es": "ancora",
  "sv": "sv_talbanken.TurkuNLP",
  "ta": "ta_ttb.mbert",
  "te": "mtg",
  "tr": "imst",
  "uk": "uk_iu.TurkuNLP",
  "hsb": "ufal",
  "ur": "udtb",
  "ug": "udt",
  "vi": "vtb",
  "wo": "wtb",
  "nb": "bokmaal",
  "mt": "mudt",
  "swl": "sslc",
  "cop": "scriptorium",
  "be": "hse",
  "zh-hans": "gsdsimp",
  "lzh": "kyoto",
  "gd": "arcosg",
  "olo": "kkpp",
  "orv": "torot",
  "nn": "nynorsk",
  "mr": "ufal"
}

