"""
Metadata about languages used by our model training code for our
SingleByteCharSetProbers.  Could be used for other things in the future.

This code was originally based on the language metadata from the uchardet
project.
"""

from dataclasses import dataclass
from string import ascii_letters


@dataclass(frozen=True)
class Language:
    """Metadata about a language useful for training models

    :ivar name: The human name for the language, in English.
    :type name: str
    :ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
                    or use another catalog as a last resort.
    :type iso_code: str
    :ivar use_ascii: Whether or not ASCII letters should be included in trained
                     models.
    :type use_ascii: bool
    :ivar charsets: The charsets we want to support and create data for.
    :type charsets: list of str
    :ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
                    `True`, you only need to add those not in the ASCII set.
    :type alphabet: str
    :ivar num_training_docs: Number of documents from CulturaX to use for training.
                             This represents approximately 300M characters of training data.
                             None means the count hasn't been determined yet.
    :type num_training_docs: int or None
    :ivar num_training_chars: Number of characters from CulturaX used for training.
                              The goal is for this to be at least 300M characters, but some
                              languages may not have that much data available.
                              None means the count hasn't been determined yet.
    :type num_training_chars: int or None
    """

    name: str
    iso_code: str
    use_ascii: bool
    charsets: list[str]
    alphabet: str
    num_training_docs: int | None = None
    num_training_chars: int | None = None

    def __repr__(self) -> str:
        param_str = ", ".join(
            f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_")
        )
        return f"{self.__class__.__name__}({param_str})"


LANGUAGES = {
    "Arabic": Language(
        name="Arabic",
        iso_code="ar",
        use_ascii=False,
        charsets=[
            "CP720",
            "CP864",
            "ISO-8859-6",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1256",
        ],
        alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ",
        num_training_docs=98_634,
        num_training_chars=300_001_734,
    ),
    "Belarusian": Language(
        name="Belarusian",
        iso_code="be",
        use_ascii=False,
        charsets=[
            "CP855",
            "CP866",
            "ISO-8859-5",
            "KOI8-R",
            "MacCyrillic",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1251",
        ],
        alphabet="АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯабвгдеёжзійклмнопрстуўфхцчшыьэюяʼ",
        num_training_docs=77_015,
        num_training_chars=300_003_525,
    ),
    "Bulgarian": Language(
        name="Bulgarian",
        iso_code="bg",
        use_ascii=False,
        charsets=[
            "CP855",
            "CP866",
            "ISO-8859-5",
            "KOI8-R",
            "MacCyrillic",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1251",
        ],
        alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя",
        num_training_docs=84_186,
        num_training_chars=300_001_428,
    ),
    "Breton": Language(
        name="Breton",
        iso_code="br",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-14",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÀÂÈÊÎÔÙÛàâèêîôùû"))),
        num_training_docs=43_761,
        num_training_chars=39_597_819,
    ),
    "Welsh": Language(
        name="Welsh",
        iso_code="cy",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-14",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        # ẀẂẄ never seem to occur in training data, so don't include them
        alphabet="".join(
            sorted(set(ascii_letters + "ÁÂÄÉÊËÍÎÏÓÔÖÚÛÜÝáâäéêëíîïóôöúûüýÿŴŵŶŷŸẁẃẅỲỳ"))
        ),
        num_training_docs=78_726,
        num_training_chars=300_014_737,
    ),
    "Czech": Language(
        name="Czech",
        iso_code="cs",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        alphabet="".join(sorted(set(ascii_letters + "áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ"))),
        num_training_docs=98_820,
        num_training_chars=300_005_076,
    ),
    "Danish": Language(
        name="Danish",
        iso_code="da",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "CP865",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "æøåÆØÅ"))),
        num_training_docs=83_466,
        num_training_chars=300_004_040,
    ),
    "German": Language(
        name="German",
        iso_code="de",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "äöüßẞÄÖÜ"))),
        num_training_docs=83_754,
        num_training_chars=300_002_572,
    ),
    "Greek": Language(
        name="Greek",
        iso_code="el",
        use_ascii=False,
        charsets=[
            "CP737",
            "CP869",
            "CP875",
            "ISO-8859-7",
            "MacGreek",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1253",
        ],
        alphabet="αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ",
        num_training_docs=103_810,
        num_training_chars=300_003_051,
    ),
    "English": Language(
        name="English",
        iso_code="en",
        alphabet=ascii_letters,
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        num_training_docs=84_511,
        num_training_chars=300_004_447,
    ),
    "Esperanto": Language(
        name="Esperanto",
        iso_code="eo",
        use_ascii=True,
        charsets=[
            "ISO-8859-3",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        # Include Q, W, X, Y for loanwords
        alphabet="".join(sorted(set(ascii_letters + "ĉĝĥĵŝŭĈĜĤĴŜŬ"))),
        num_training_docs=40_441,
        num_training_chars=300_001_893,
    ),
    "Spanish": Language(
        name="Spanish",
        iso_code="es",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ñáéíóúüÑÁÉÍÓÚÜ"))),
        num_training_docs=87_069,
        num_training_chars=300_000_884,
    ),
    "Estonian": Language(
        name="Estonian",
        iso_code="et",
        use_ascii=True,
        charsets=[
            "CP775",
            "ISO-8859-4",
            "ISO-8859-13",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1257",
        ],
        # Include C, F, Š, Q, W, X, Y, Z, Ž for loanwords
        alphabet="".join(sorted(set(ascii_letters + "õäöüšžÕÄÖÜŠŽ"))),
        num_training_docs=66_818,
        num_training_chars=300_000_765,
    ),
    "Farsi": Language(
        name="Farsi",
        iso_code="fa",
        use_ascii=False,
        charsets=[
            "CP720",
            "CP864",
            "ISO-8859-6",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1256",
        ],
        alphabet="ءآأؤإئابپتثجچحخدذرزژسشصضطظعغفقکگلمنوهیًٌٍَُِّ",
        num_training_docs=104_835,
        num_training_chars=300_001_684,
    ),
    "Finnish": Language(
        name="Finnish",
        iso_code="fi",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÅÄÖŠŽåäöšž"))),
        num_training_docs=80_361,
        num_training_chars=300_001_375,
    ),
    "French": Language(
        name="French",
        iso_code="fr",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "CP863",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(
            sorted(set(ascii_letters + "àâæçèéêëîïôùûüÿÀÂÆÇÈÉÊËÎÏÔÙÛÜŸŒœ"))
        ),
        num_training_docs=88_369,
        num_training_chars=300_003_422,
    ),
    "Irish": Language(
        name="Irish",
        iso_code="ga",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-14",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÁÉÍÓÚáéíóú"))),
        num_training_docs=63_468,
        num_training_chars=300_005_169,
    ),
    "Scottish Gaelic": Language(
        name="Scottish Gaelic",
        iso_code="gd",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-14",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÀÈÌÒÙàèìòù"))),
        num_training_docs=8_408,
        num_training_chars=15_116_797,
    ),
    "Hebrew": Language(
        name="Hebrew",
        iso_code="he",
        use_ascii=False,
        charsets=[
            "CP424",
            "CP856",
            "CP862",
            "ISO-8859-8",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1255",
        ],
        alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ",
        num_training_docs=58265,
        num_training_chars=300_009_414,
    ),
    "Croatian": Language(
        name="Croatian",
        iso_code="hr",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        # Include Q, W, X, Y for loanwords
        alphabet="".join(sorted(set(ascii_letters + "čćđšžČĆĐŠŽ"))),
        num_training_docs=460_689,
        num_training_chars=157_579_665,
    ),
    "Hungarian": Language(
        name="Hungarian",
        iso_code="hu",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        # Include Q, W, X, Y for loanwords
        alphabet="".join(sorted(set(ascii_letters + "áéíóöőúüűÁÉÍÓÖŐÚÜŰ"))),
        num_training_docs=82_417,
        num_training_chars=300_001_846,
    ),
    "Icelandic": Language(
        name="Icelandic",
        iso_code="is",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "CP861",
            "ISO-8859-1",
            "ISO-8859-10",
            "ISO-8859-15",
            "MacIceland",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        # Include Q, W for loanwords
        alphabet="".join(sorted(set(ascii_letters + "áðéíóúýþæöÁÐÉÍÓÚÝÞÆÖ"))),
        num_training_docs=77_487,
        num_training_chars=300_004_354,
    ),
    "Indonesian": Language(
        name="Indonesian",
        iso_code="id",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet=ascii_letters,
        num_training_docs=114_888,
        num_training_chars=300_000_301,
    ),
    "Italian": Language(
        name="Italian",
        iso_code="it",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÀÈÉÌÒÓÙàèéìòóù"))),
        num_training_docs=92_388,
        num_training_chars=300_081_924,
    ),
    "Japanese": Language(
        name="Japanese",
        iso_code="ja",
        use_ascii=False,
        charsets=[
            "CP932",
            "EUC-JP",
            "ISO-2022-JP",
            "SHIFT-JIS",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="",
        num_training_docs=None,
        num_training_chars=None,
    ),
    "Kazakh": Language(
        name="Kazakh",
        iso_code="kk",
        use_ascii=False,
        charsets=[
            "KZ1048",
            "PTCP154",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="АӘБВГҒДЕЁЖЗИЙКҚЛМНҢОӨПРСТУҰҮФХҺЦЧШЩЪЫІЬЭЮЯаәбвгғдеёжзийкқлмнңоөпрстууұүфхһцчшщъыіьэюя",
        num_training_docs=73_160,
        num_training_chars=300_002_618,
    ),
    "Korean": Language(
        name="Korean",
        iso_code="ko",
        use_ascii=False,
        charsets=[
            "CP949",
            "EUC-KR",
            "ISO-2022-KR",
            "JOHAB",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="",
        num_training_docs=None,
        num_training_chars=None,
    ),
    "Cornish": Language(
        name="Cornish",
        iso_code="kw",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-14",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÂÊÎÔÛâêîôûŴŵŶŷ"))),
        num_training_docs=94,
        num_training_chars=58_047,
    ),
    "Lithuanian": Language(
        name="Lithuanian",
        iso_code="lt",
        use_ascii=True,
        charsets=[
            "CP775",
            "ISO-8859-4",
            "ISO-8859-13",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1257",
        ],
        # Include Q, W, X for loanwords
        alphabet="".join(sorted(set(ascii_letters + "ąčęėįšųūžĄČĘĖĮŠŲŪŽ"))),
        num_training_docs=73_445,
        num_training_chars=300_008_498,
    ),
    "Latvian": Language(
        name="Latvian",
        iso_code="lv",
        use_ascii=True,
        charsets=[
            "CP775",
            "ISO-8859-4",
            "ISO-8859-13",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1257",
        ],
        # Include Q, W, X, Y for loanwords
        alphabet="".join(sorted(set(ascii_letters + "āčēģīķļņšūžĀČĒĢĪĶĻŅŠŪŽ"))),
        num_training_docs=71_628,
        num_training_chars=300_007_767,
    ),
    "Macedonian": Language(
        name="Macedonian",
        iso_code="mk",
        use_ascii=False,
        charsets=[
            "CP855",
            "CP866",
            "ISO-8859-5",
            "KOI8-R",
            "MacCyrillic",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1251",
        ],
        alphabet="АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШабвгдѓежзѕијклљмнњопрстќуфхцчџш",
        num_training_docs=102_724,
        num_training_chars=300_009_729,
    ),
    "Maltese": Language(
        name="Maltese",
        iso_code="mt",
        use_ascii=True,
        charsets=[
            "ISO-8859-3",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        # Include Y for loanwords
        alphabet="".join(sorted(set(ascii_letters + "ċġħżĊĠĦŻ"))),
        num_training_docs=51_488,
        num_training_chars=300_001_960,
    ),
    "Malay": Language(
        name="Malay",
        iso_code="ms",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet=ascii_letters,
        num_training_docs=238_150,
        num_training_chars=93_983_043,
    ),
    "Dutch": Language(
        name="Dutch",
        iso_code="nl",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        # Include loanword characters commonly used in Dutch
        alphabet="".join(sorted(set(ascii_letters + "àâçèéêëïîñôùûœÀÂÇÈÉÊËÏÎÑÔÙÛŒ"))),
        num_training_docs=107_675,
        num_training_chars=300_000_260,
    ),
    "Norwegian": Language(
        name="Norwegian",
        iso_code="no",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "CP865",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        # Include Q, W, X, Z and common loanword characters
        alphabet="".join(sorted(set(ascii_letters + "ÆØÅæøå"))),
        num_training_docs=66_762,
        num_training_chars=300_001_076,
    ),
    "Polish": Language(
        name="Polish",
        iso_code="pl",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        # Include Q, X for loanwords
        alphabet="".join(sorted(set(ascii_letters + "ąćęłńóśźżĄĆĘŁŃÓŚŹŻ"))),
        num_training_docs=97_060,
        num_training_chars=300_001_942,
    ),
    "Portuguese": Language(
        name="Portuguese",
        iso_code="pt",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "CP860",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú"))),
        num_training_docs=101_817,
        num_training_chars=300_001_295,
    ),
    "Romanian": Language(
        name="Romanian",
        iso_code="ro",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        alphabet="".join(sorted(set(ascii_letters + "ăâîșțĂÂÎȘȚ"))),
        num_training_docs=78_976,
        num_training_chars=300_001_970,
    ),
    "Russian": Language(
        name="Russian",
        iso_code="ru",
        use_ascii=False,
        charsets=[
            "CP855",
            "CP866",
            "ISO-8859-5",
            "KOI8-R",
            "MacCyrillic",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1251",
        ],
        alphabet="абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ",
        num_training_docs=85_054,
        num_training_chars=300_001_344,
    ),
    "Slovak": Language(
        name="Slovak",
        iso_code="sk",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        alphabet="".join(
            sorted(set(ascii_letters + "áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ"))
        ),
        num_training_docs=95_223,
        num_training_chars=300_000_329,
    ),
    "Slovene": Language(
        name="Slovene",
        iso_code="sl",
        use_ascii=True,
        charsets=[
            "CP852",
            "ISO-8859-2",
            "ISO-8859-16",
            "MacLatin2",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1250",
        ],
        # Include Q, W, X, Y for loanwords
        alphabet="".join(sorted(set(ascii_letters + "čšžČŠŽ"))),
        num_training_docs=66_688,
        num_training_chars=300_002_768,
    ),
    # Serbian can be written in both Latin and Cyrillic, but there's no
    # simple way to get the Latin alphabet pages from Wikipedia through
    # the API, so for now we just support Cyrillic. The Latin alphabet
    # is the same as Croatian, so we reuse that language model to handle
    # Latin-Serbian text.
    "Serbian": Language(
        name="Serbian",
        iso_code="sr",
        use_ascii=False,
        alphabet="АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШабвгдђежзијклљмнњопрстћуфхцчџш",
        charsets=[
            "CP855",
            "CP866",
            "ISO-8859-5",
            "KOI8-R",
            "MacCyrillic",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1251",
        ],
        num_training_docs=66_918,
        num_training_chars=300_000_904,
    ),
    "Swedish": Language(
        name="Swedish",
        iso_code="sv",
        use_ascii=True,
        charsets=[
            "CP037",
            "CP437",
            "CP500",
            "CP850",
            "CP858",
            "ISO-8859-1",
            "ISO-8859-15",
            "MacRoman",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1252",
        ],
        # Include Q, W, Z (loanwords) and common accented characters
        alphabet="".join(sorted(set(ascii_letters + "ÅÄÖåäö"))),
        num_training_docs=96_485,
        num_training_chars=300_013_381,
    ),
    "Tajik": Language(
        name="Tajik",
        iso_code="tg",
        use_ascii=False,
        charsets=[
            "KOI8-T",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="АБВГҒДЕЁЖЗИӢЙКҚЛМНОПРСТУӮФХҲЧҶШЪЭЮЯабвгғдеёжзиӣйкқлмнопрстуӯфхҳчҷшъэюя",
        num_training_docs=74_865,
        num_training_chars=300_022_133,
    ),
    "Thai": Language(
        name="Thai",
        iso_code="th",
        use_ascii=False,
        charsets=[
            "CP874",
            "ISO-8859-11",
            "TIS-620",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛",
        num_training_docs=109_133,
        num_training_chars=300_008_106,
    ),
    "Turkish": Language(
        name="Turkish",
        iso_code="tr",
        use_ascii=True,
        charsets=[
            "CP857",
            "CP1026",
            "ISO-8859-3",
            "ISO-8859-9",
            "MacTurkish",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1254",
        ],
        # Include Q, W, X for loanwords
        alphabet="".join(sorted(set(ascii_letters + "çğıiöşüâîûÇĞIİÖŞÜÂÎÛ"))),
        num_training_docs=107_848,
        num_training_chars=300_001_308,
    ),
    "Ukrainian": Language(
        name="Ukrainian",
        iso_code="uk",
        use_ascii=False,
        charsets=[
            "CP855",
            "CP866",
            "CP1125",
            "ISO-8859-5",
            "KOI8-R",
            "KOI8-U",
            "MacCyrillic",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1251",
        ],
        alphabet="АБВГҐДЕЄЖЗИІЇЙКЛМНОПРСТУФХЦЧШЩЬЮЯабвгґдеєжзиіїйклмнопрстуфхцчшщьюяʼ",
        num_training_docs=95_019,
        num_training_chars=300_048_946,
    ),
    "Urdu": Language(
        name="Urdu",
        iso_code="ur",
        use_ascii=False,
        charsets=[
            "CP1006",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيپچڈڑژکگںھۂۃیے",
        num_training_docs=82_296,
        num_training_chars=300_001_141,
    ),
    "Vietnamese": Language(
        name="Vietnamese",
        iso_code="vi",
        use_ascii=False,
        # Windows-1258 is the only common 8-bit
        # Vietnamese encoding supported by Python.
        # From Wikipedia:
        # For systems that lack support for Unicode,
        # dozens of 8-bit Vietnamese code pages are
        # available.[1] The most common are VISCII
        # (TCVN 5712:1993), VPS, and Windows-1258.[3]
        # Where ASCII is required, such as when
        # ensuring readability in plain text e-mail,
        # Vietnamese letters are often encoded
        # according to Vietnamese Quoted-Readable
        # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
        # though usage of either variable-width
        # scheme has declined dramatically following
        # the adoption of Unicode on the World Wide
        # Web.
        charsets=[
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
            "WINDOWS-1258",
        ],
        # Alphabet includes base letters and combining tone marks used by Windows-1258:
        # - Base letters: a, ă, â, e, ê, o, ô, ơ, u, ư, etc.
        # - Combining marks: grave (̀), acute (́), tilde (̃), hook above (̉), dot below (̣)
        alphabet="aăâbcdđeêghiklmnoôơpqrstuưvxyAĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY\u0300\u0301\u0303\u0309\u0323",
        num_training_docs=85_692,
        num_training_chars=300_000_871,
    ),
    "Chinese": Language(
        name="Chinese",
        iso_code="zh",
        use_ascii=False,
        charsets=[
            "BIG5",
            "GB18030",
            "HZ-GB-2312",
            "UTF-8",
            "UTF-16",
            "UTF-16BE",
            "UTF-16LE",
            "UTF-32",
            "UTF-32BE",
            "UTF-32LE",
        ],
        alphabet="",
        num_training_docs=None,
        num_training_chars=None,
    ),
}
