{
  "metadata": {
    "tokenizers": {
      "gpt4_cl100k": "GPT-4 (cl100k_base)",
      "gpt4o_o200k": "GPT-4o (o200k_base)",
      "llama3": "Llama 3",
      "gemma": "Gemma 2",
      "mistral": "Mistral v0.3",
      "bert_multilingual": "BERT Multilingual",
      "xlm_roberta": "XLM-RoBERTa"
    },
    "languages": {
      "eng": "English",
      "yor": "Yorùbá",
      "hau": "Hausa",
      "ibo": "Igbo",
      "pcm": "Pidgin"
    },
    "gpt4o_pricing_usd_per_1m": 2.5,
    "usd_to_ngn_rate": 1600.0
  },
  "results": [
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        " morning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        " morning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        "Ġmorning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        "▁morning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Good",
        "▁morning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        "morning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Good morning",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Good",
        "▁morning"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "�",
        "�",
        " k",
        "ú",
        " à",
        "ár",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Ẹ",
        " kú",
        " à",
        "ár",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "áº",
        "¸",
        "Ġk",
        "Ãº",
        "ĠÃł",
        "Ã¡r",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Ẹ",
        "▁kú",
        "▁à",
        "ár",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "▁k",
        "ú",
        "▁à",
        "ár",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Ẹ",
        "k",
        "##ú",
        "à",
        "##ár",
        "##ọ̀"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú àárọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁",
        "Ẹ",
        "▁kú",
        "▁à",
        "ár",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "In",
        "a",
        " kw",
        "ana"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ina",
        " kw",
        "ana"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "In",
        "a",
        "Ġkw",
        "ana"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Ina",
        "▁kw",
        "ana"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁In",
        "a",
        "▁kw",
        "ana"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ina",
        "kwa",
        "##na"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina kwana",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Ina",
        "▁kwa",
        "na"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "�",
        "�",
        "̀",
        "t",
        "ụ",
        "̀",
        "t",
        "ụ",
        "̀",
        " ",
        "ọ",
        "̀",
        "ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 6.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Ụ",
        "̀",
        "tụ",
        "̀",
        "tụ",
        "̀",
        " ọ",
        "̀",
        "ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "á»",
        "¤",
        "ÌĢ",
        "t",
        "á»¥",
        "ÌĢ",
        "t",
        "á»¥",
        "ÌĢ",
        "Ġ",
        "á»į",
        "ÌĢ",
        "ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 6.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "Ụ",
        "̀",
        "t",
        "ụ",
        "̀",
        "t",
        "ụ",
        "̀",
        "▁ọ",
        "̀",
        "ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0xA4>",
        "̀",
        "t",
        "ụ",
        "̀",
        "t",
        "ụ",
        "̀",
        "▁",
        "ọ",
        "̀",
        "ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 7.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "[UNK]",
        "ọ",
        "##̀",
        "##ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụ̀tụ̀tụ̀ ọ̀ma",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁",
        "Ụ",
        "̀",
        "t",
        "ụ",
        "̀",
        "t",
        "ụ",
        "̀",
        "▁",
        "ọ",
        "̀",
        "ma"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 6.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Good",
        " morning",
        " o"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Good",
        " morning",
        " o"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Good",
        "Ġmorning",
        "Ġo"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Good",
        "▁morning",
        "▁o"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Good",
        "▁morning",
        "▁o"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Good",
        "morning",
        "o"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 1,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Good morning o",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Good",
        "▁morning",
        "▁o"
      ],
      "english_equivalent": "Good morning",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "How",
        " are",
        " you",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "How",
        " are",
        " you",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "How",
        "Ġare",
        "Ġyou",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "How",
        "▁are",
        "▁you",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁How",
        "▁are",
        "▁you",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "How",
        "are",
        "you",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "How are you?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁How",
        "▁are",
        "▁you",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "�",
        "�",
        "�",
        "é",
        " à",
        "l",
        "à",
        "áf",
        "í",
        "à",
        " ni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Ṣ",
        "é",
        " à",
        "l",
        "à",
        "á",
        "f",
        "í",
        "à",
        " ni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "á¹",
        "¢",
        "Ã©",
        "ĠÃł",
        "l",
        "Ãł",
        "Ã¡f",
        "ÃŃ",
        "Ãł",
        "Ġni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "Ṣ",
        "é",
        "▁à",
        "là",
        "á",
        "fí",
        "à",
        "▁ni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁",
        "Ṣ",
        "é",
        "▁à",
        "l",
        "à",
        "á",
        "f",
        "í",
        "à",
        "▁ni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "Ṣ",
        "##é",
        "à",
        "##là",
        "##á",
        "##f",
        "##í",
        "##à",
        "ni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ṣé àlàáfíà ni?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁",
        "Ṣ",
        "é",
        "▁à",
        "là",
        "á",
        "fí",
        "à",
        "▁ni",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Y",
        "aya",
        " k",
        "ake",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Y",
        "aya",
        " k",
        "ake",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Y",
        "aya",
        "Ġk",
        "ake",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Y",
        "aya",
        "▁k",
        "ake",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Y",
        "aya",
        "▁k",
        "ake",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ya",
        "##ya",
        "ka",
        "##ke",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaya kake?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Ya",
        "ya",
        "▁ka",
        "ke",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "K",
        "ed",
        "ụ",
        " ka",
        " ",
        "ị",
        " mere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "K",
        "ed",
        "ụ",
        " ka",
        " ị",
        " mere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "K",
        "ed",
        "á»¥",
        "Ġka",
        "Ġ",
        "á»ĭ",
        "Ġmere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 7,
      "token_splits": [
        "Ked",
        "ụ",
        "▁ka",
        "▁",
        "ị",
        "▁mere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁K",
        "ed",
        "ụ",
        "▁ka",
        "▁",
        "ị",
        "▁mere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "Ke",
        "##d",
        "##ụ",
        "ka",
        "ị",
        "mere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Kedụ ka ị mere?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 8,
      "token_splits": [
        "▁K",
        "ed",
        "ụ",
        "▁ka",
        "▁",
        "ị",
        "▁mere",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "How",
        " you",
        " de",
        "y",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "How",
        " you",
        " dey",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "How",
        "Ġyou",
        "Ġde",
        "y",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "How",
        "▁you",
        "▁dey",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁How",
        "▁you",
        "▁de",
        "y",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "How",
        "you",
        "de",
        "##y",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 2,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How you dey?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁How",
        "▁you",
        "▁de",
        "y",
        "?"
      ],
      "english_equivalent": "How are you?",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        " you",
        " very",
        " much"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        " you",
        " very",
        " much"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        "Ġyou",
        "Ġvery",
        "Ġmuch"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        "▁you",
        "▁very",
        "▁much"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Thank",
        "▁you",
        "▁very",
        "▁much"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        "you",
        "very",
        "much"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Thank you very much",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Thank",
        "▁you",
        "▁very",
        "▁much"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "�",
        "�",
        " �",
        "�",
        "�",
        "é",
        " p",
        "ú",
        "p",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Ẹ",
        " ṣ",
        "é",
        " pú",
        "pọ",
        "̀"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "áº",
        "¸",
        "Ġá",
        "¹",
        "£",
        "Ã©",
        "Ġp",
        "Ãºp",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "Ẹ",
        "▁",
        "ṣ",
        "é",
        "▁p",
        "úp",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "▁",
        "ṣ",
        "é",
        "▁p",
        "ú",
        "p",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Ẹ",
        "ṣ",
        "##é",
        "p",
        "##úp",
        "##ọ̀"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ ṣé púpọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "Ẹ",
        "▁",
        "ṣ",
        "é",
        "▁pú",
        "p",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Na",
        " gode",
        " sos",
        "ai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Na",
        " gode",
        " sosai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Na",
        "Ġgode",
        "Ġsos",
        "ai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Na",
        "▁gode",
        "▁sos",
        "ai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁Na",
        "▁g",
        "ode",
        "▁s",
        "os",
        "ai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Na",
        "gode",
        "sos",
        "##ai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Na gode sosai",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Na",
        "▁gode",
        "▁so",
        "sai"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "Da",
        "al",
        "ụ",
        " n",
        "ke",
        " uk",
        "w",
        "uu"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "D",
        "aal",
        "ụ",
        " nke",
        " ukwuu"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "Da",
        "al",
        "á»¥",
        "Ġn",
        "ke",
        "Ġuk",
        "w",
        "uu"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 7,
      "token_splits": [
        "Da",
        "al",
        "ụ",
        "▁nke",
        "▁uk",
        "w",
        "uu"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁Da",
        "al",
        "ụ",
        "▁n",
        "ke",
        "▁u",
        "kw",
        "uu"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 8,
      "token_splits": [
        "Da",
        "##al",
        "##ụ",
        "n",
        "##ke",
        "uk",
        "##wu",
        "##u"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Daalụ nke ukwuu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 8,
      "token_splits": [
        "▁Da",
        "al",
        "ụ",
        "▁",
        "nke",
        "▁uk",
        "wu",
        "u"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        " you",
        " well",
        " well"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        " you",
        " well",
        " well"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        "Ġyou",
        "Ġwell",
        "Ġwell"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        "▁you",
        "▁well",
        "▁well"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Thank",
        "▁you",
        "▁well",
        "▁well"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Thank",
        "you",
        "well",
        "well"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 3,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Thank you well well",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Thank",
        "▁you",
        "▁well",
        "▁well"
      ],
      "english_equivalent": "Thank you very much",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Welcome",
        " home"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Welcome",
        " home"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Welcome",
        "Ġhome"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Welcome",
        "▁home"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Welcome",
        "▁home"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Welcome",
        "home"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Welcome home",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Welcome",
        "▁home"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "�",
        "�",
        " k",
        "ú",
        " il",
        "é",
        " o"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ẹ",
        " kú",
        " il",
        "é",
        " o"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "áº",
        "¸",
        "Ġk",
        "Ãº",
        "Ġil",
        "Ã©",
        "Ġo"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ẹ",
        "▁kú",
        "▁il",
        "é",
        "▁o"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "▁k",
        "ú",
        "▁il",
        "é",
        "▁o"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Ẹ",
        "k",
        "##ú",
        "il",
        "##é",
        "o"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ilé o",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "Ẹ",
        "▁kú",
        "▁i",
        "lé",
        "▁o"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "S",
        "ann",
        "u",
        " da",
        " zu",
        "wa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "S",
        "annu",
        " da",
        " zuwa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "S",
        "ann",
        "u",
        "Ġda",
        "Ġzu",
        "wa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "S",
        "annu",
        "▁da",
        "▁zu",
        "wa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁S",
        "ann",
        "u",
        "▁da",
        "▁zu",
        "wa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "San",
        "##nu",
        "da",
        "zu",
        "##wa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sannu da zuwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁San",
        "nu",
        "▁da",
        "▁zuwa"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "N",
        "n",
        "ọ",
        "ọ",
        " n",
        "'",
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "N",
        "n",
        "ọọ",
        " n",
        "'ụ",
        "lọ"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "N",
        "n",
        "á»į",
        "á»į",
        "Ġn",
        "'",
        "á»¥",
        "l",
        "á»į"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "Nn",
        "ọ",
        "ọ",
        "▁n",
        "'",
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁N",
        "n",
        "ọ",
        "ọ",
        "▁n",
        "'",
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "N",
        "##n",
        "##ọ",
        "##ọ",
        "n",
        "'",
        "ụ",
        "##l",
        "##ọ"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnọọ n'ụlọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁N",
        "n",
        "ọ",
        "ọ",
        "▁n",
        "'",
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "You",
        " don",
        " come",
        " house"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "You",
        " don",
        " come",
        " house"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "You",
        "Ġdon",
        "Ġcome",
        "Ġhouse"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "You",
        "▁don",
        "▁come",
        "▁house"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁You",
        "▁don",
        "▁come",
        "▁house"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "You",
        "don",
        "come",
        "house"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 4,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "You don come house",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁You",
        "▁don",
        "▁come",
        "▁house"
      ],
      "english_equivalent": "Welcome home",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        "bye"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        "bye"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Good",
        "bye"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Goodbye"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Good",
        "bye"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "Goodbye"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Goodbye",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Good",
        "by",
        "e"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Ó",
        " d",
        "à",
        "b",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ó",
        " dà",
        "bọ",
        "̀"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "Ãĵ",
        "Ġd",
        "Ãł",
        "b",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ó",
        "▁dà",
        "b",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁",
        "Ó",
        "▁d",
        "à",
        "b",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ó",
        "dà",
        "##b",
        "##ọ̀"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ó dàbọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Ó",
        "▁dà",
        "b",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "S",
        "ai",
        " an",
        " j",
        "ima"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Sai",
        " an",
        " j",
        "ima"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "S",
        "ai",
        "Ġan",
        "Ġj",
        "ima"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Sai",
        "▁an",
        "▁j",
        "ima"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁S",
        "ai",
        "▁an",
        "▁j",
        "ima"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Sai",
        "an",
        "jim",
        "##a"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sai an jima",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Sai",
        "▁an",
        "▁",
        "jima"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ka",
        " ",
        "ọ",
        " d",
        "ị"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ka",
        " ọ",
        " dị"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ka",
        "Ġ",
        "á»į",
        "Ġdá»ĭ"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Ka",
        "▁ọ",
        "▁dị"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Ka",
        "▁",
        "ọ",
        "▁d",
        "ị"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ka",
        "ọ",
        "d",
        "##ị"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ka ọ dị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Ka",
        "▁",
        "ọ",
        "▁dị"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Make",
        " we",
        " de",
        "y",
        " go"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Make",
        " we",
        " dey",
        " go"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Make",
        "Ġwe",
        "Ġde",
        "y",
        "Ġgo"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Make",
        "▁we",
        "▁dey",
        "▁go"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Make",
        "▁we",
        "▁de",
        "y",
        "▁go"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Make",
        "we",
        "de",
        "##y",
        "go"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 5,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Make we dey go",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Make",
        "▁we",
        "▁de",
        "y",
        "▁go"
      ],
      "english_equivalent": "Goodbye",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "water",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "omi"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "omi"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "omi"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "omi"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁om",
        "i"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "om",
        "##i"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "omi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁om",
        "i"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ru",
        "wa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ru",
        "wa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ru",
        "wa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ru",
        "wa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁ru",
        "wa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ru",
        "##wa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁ruwa"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "mm",
        "ị",
        "r",
        "ị"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "mm",
        "ịrị"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "mm",
        "á»ĭ",
        "r",
        "á»ĭ"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "mm",
        "ị",
        "r",
        "ị"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁mm",
        "ị",
        "r",
        "ị"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "mm",
        "##ị",
        "##r",
        "##ị"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁mm",
        "ị",
        "r",
        "ị"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 6,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "water",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁water"
      ],
      "english_equivalent": "water",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "food",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁food"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "o",
        "ún",
        "j",
        "�",
        "�"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "o",
        "ún",
        "jẹ"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "o",
        "Ãºn",
        "j",
        "áº¹"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "o",
        "ún",
        "j",
        "ẹ"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁o",
        "ún",
        "j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "o",
        "##ún",
        "##j",
        "##ẹ"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oúnjẹ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁o",
        "ún",
        "j",
        "ẹ"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "inci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "inci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "inci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "inci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁ab",
        "in",
        "ci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "##inci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "abinci",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁abin",
        "ci"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "ri"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "ri"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "ri"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "ri"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁n",
        "ri"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "nr",
        "##i"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁nr",
        "i"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ch",
        "op"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ch",
        "op"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "chop"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "chop"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁chop"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "cho",
        "##p"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 7,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "chop"
      ],
      "english_equivalent": "food",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "money",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ow",
        "ó"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ow",
        "ó"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ow",
        "Ã³"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ow",
        "ó"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁o",
        "w",
        "ó"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "o",
        "##w",
        "##ó"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "owó",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ow",
        "ó"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ku",
        "�",
        "�",
        "i"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ku",
        "ɗ",
        "i"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ku",
        "É",
        "Ĺi"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ku",
        "ɗ",
        "i"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁k",
        "u",
        "<0xC9>",
        "<0x97>",
        "i"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "[UNK]"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kuɗi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ku",
        "ɗ",
        "i"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ego",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁ego"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 8,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "money",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁money"
      ],
      "english_equivalent": "money",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "house",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "il",
        "é"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ilé"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "il",
        "Ã©"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "ilé"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁il",
        "é"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "il",
        "##é"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁i",
        "lé"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "ida"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "ida"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "ida"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "gida"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁g",
        "ida"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "gi",
        "##da"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gida",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁gida"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ụlọ"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "á»¥",
        "l",
        "á»į"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ụ",
        "##l",
        "##ọ"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ụ",
        "l",
        "ọ"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 9,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "house",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁house"
      ],
      "english_equivalent": "house",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "child",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁child"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "m",
        "ọ"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "mọ"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "á»į",
        "m",
        "á»į"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "m",
        "ọ"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ọ",
        "m",
        "ọ"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "##m",
        "##ọ"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọmọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ọ",
        "m",
        "ọ"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "aro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "aro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "aro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "aro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁y",
        "aro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ya",
        "##ro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yaro",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ya",
        "ro"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁n",
        "wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "##wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁n",
        "wa"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "ik",
        "in"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "ikin"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "ik",
        "in"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "ikin"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁p",
        "ik",
        "in"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "pi",
        "##kin"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 10,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pikin",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁pik",
        "in"
      ],
      "english_equivalent": "child",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "mother",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁mother"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ì",
        "y",
        "á"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ì",
        "y",
        "á"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Ã¬",
        "y",
        "Ã¡"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ì",
        "y",
        "á"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ì",
        "y",
        "á"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ì",
        "##y",
        "##á"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ìyá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ì",
        "y",
        "á"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "u",
        "wa"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "uwa"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "u",
        "wa"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "uwa"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁u",
        "wa"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "u",
        "##wa"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁uw",
        "a"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "ne"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "nne"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "ne"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "nne"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁n",
        "ne"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "nn",
        "##e"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nne",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "nne"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "m",
        "ama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "m",
        "ama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "m",
        "ama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "mama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁m",
        "ama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "mama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 11,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "mama",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁mama"
      ],
      "english_equivalent": "mother",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "father",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁father"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "b",
        "à",
        "b",
        "á"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "b",
        "à",
        "b",
        "á"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "b",
        "Ãł",
        "b",
        "Ã¡"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "b",
        "à",
        "bá"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁b",
        "à",
        "b",
        "á"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "bà",
        "##bá"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "bàbá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁bà",
        "bá"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "uba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "uba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "uba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "uba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁u",
        "ba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "u",
        "##ba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "uba",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁u",
        "ba"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "na"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "na"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "na"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "n",
        "na"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁n",
        "na"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "nn",
        "##a"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "nna",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "nna"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "apa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "apa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "apa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "papa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁p",
        "apa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "papa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 12,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "papa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁papa"
      ],
      "english_equivalent": "father",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "road",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "á»į",
        "ÌĢ",
        "n",
        "Ãł"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "̀n",
        "à"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "##̀",
        "##n",
        "##à"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀nà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "h",
        "anya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "h",
        "anya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "h",
        "anya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "h",
        "anya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁h",
        "anya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "hanya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hanya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁hanya"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ụ",
        "zọ"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "á»¥",
        "z",
        "á»į"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ụ",
        "##z",
        "##ọ"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụzọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 13,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "road",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁road"
      ],
      "english_equivalent": "road",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "market",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "jà"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "jà"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "á»į",
        "jÃł"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "jà"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ọ",
        "jà"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "##jà"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọjà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ọ",
        "j",
        "à"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "kas",
        "u",
        "wa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "asu",
        "wa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "kas",
        "u",
        "wa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "kas",
        "uwa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁k",
        "as",
        "u",
        "wa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "kas",
        "##uwa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kasuwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁kasuwa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ah",
        "ị",
        "a"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ahịa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ah",
        "á»ĭa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ah",
        "ịa"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁ah",
        "ị",
        "a"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ah",
        "##ị",
        "##a"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ahịa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ah",
        "ị",
        "a"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 14,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "market",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁market"
      ],
      "english_equivalent": "market",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "king",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "ba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ọba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "á»į",
        "ba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "ba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ọ",
        "ba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "##ba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọba",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ọ",
        "ba"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "ark",
        "i"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "s",
        "arki"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "ark",
        "i"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "s",
        "arki"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁s",
        "ark",
        "i"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "sa",
        "##rki"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sarki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁sark",
        "i"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "e",
        "ze"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "e",
        "ze"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "e",
        "ze"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "eze"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁e",
        "ze"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ez",
        "##e"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "eze",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "eze"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 15,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "king",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁king"
      ],
      "english_equivalent": "king",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " eat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " eat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġeat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁eat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁eat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "eat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to eat",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁eat"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "j",
        "�",
        "�",
        "un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "jẹ",
        "un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "j",
        "áº¹",
        "un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "j",
        "ẹ",
        "un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "j",
        "##ẹ",
        "##un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jẹun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁j",
        "ẹ",
        "un"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ci",
        " ab",
        "inci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ci",
        " ab",
        "inci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ci",
        "Ġab",
        "inci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ci",
        "▁ab",
        "inci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁ci",
        "▁ab",
        "in",
        "ci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ci",
        "ab",
        "##inci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ci abinci",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ci",
        "▁abin",
        "ci"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "iri",
        " n",
        "ri"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "iri",
        " nri"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "iri",
        "Ġn",
        "ri"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "iri",
        "▁n",
        "ri"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁ir",
        "i",
        "▁n",
        "ri"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "iri",
        "nr",
        "##i"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "iri nri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁ir",
        "i",
        "▁nr",
        "i"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ch",
        "op"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ch",
        "op"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "chop"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "chop"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁chop"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "cho",
        "##p"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 16,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "chop",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "chop"
      ],
      "english_equivalent": "to eat",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġgo"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to go",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "l",
        "ọ"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "lọ"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "l",
        "á»į"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "l",
        "ọ"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁l",
        "ọ"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "l",
        "##ọ"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁l",
        "ọ"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "fi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "fi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "fi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "fi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁t",
        "afi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "##fi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tafi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ta",
        "fi"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ị",
        "ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ị",
        "ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "á»ĭ",
        "ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ị",
        "ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ị",
        "ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ị",
        "##ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ịga",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ị",
        "ga"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 17,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "go",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁go"
      ],
      "english_equivalent": "to go",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġcome"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to come",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "w",
        "á"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "w",
        "á"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "w",
        "Ã¡"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "w",
        "á"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁w",
        "á"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "w",
        "##á"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "wá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁w",
        "á"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁zo"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "b",
        "ị",
        "a"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "b",
        "ịa"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "b",
        "á»ĭa"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "b",
        "ịa"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁b",
        "ị",
        "a"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "bị",
        "##a"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "bịa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁bị",
        "a"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 18,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "come",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁come"
      ],
      "english_equivalent": "to come",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " speak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " speak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġspeak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁speak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁speak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "speak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to speak",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁speak"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "sọ",
        "̀",
        "rọ",
        "̀"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "s",
        "á»į",
        "ÌĢ",
        "r",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "s",
        "##ọ̀",
        "##r",
        "##ọ̀"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sọ̀rọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        " mag",
        "ana"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "yi",
        " magana"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "Ġmag",
        "ana"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "▁ma",
        "gana"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁y",
        "i",
        "▁mag",
        "ana"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "maga",
        "##na"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi magana",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁yi",
        "▁maga",
        "na"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "kw",
        "uo",
        " ok",
        "wu"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "kw",
        "uo",
        " okwu"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "kw",
        "uo",
        "Ġok",
        "wu"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "kw",
        "uo",
        "▁ok",
        "wu"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁kw",
        "uo",
        "▁ok",
        "w",
        "u"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "##wu",
        "##o",
        "ok",
        "##wu"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwuo okwu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁kw",
        "u",
        "o",
        "▁ok",
        "wu"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 19,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "talk",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁talk"
      ],
      "english_equivalent": "to speak",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġbuy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to buy",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ra",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁ra"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "s",
        "aya"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "s",
        "aya"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "s",
        "aya"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "saya"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁say",
        "a"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "saya"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "saya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁saya"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "ụ",
        "ta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "z",
        "ụta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "á»¥",
        "ta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "ụ",
        "ta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁z",
        "ụ",
        "ta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "##ụ",
        "##ta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "zụta",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁z",
        "ụ",
        "ta"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 20,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "buy",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁buy"
      ],
      "english_equivalent": "to buy",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġwork"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to work",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "�",
        "�",
        "�",
        "i",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ṣiṣẹ",
        "́"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "á¹",
        "£i",
        "á¹",
        "£",
        "áº¹",
        "Ìģ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ṣ",
        "i",
        "ṣ",
        "ẹ",
        "́"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ṣ",
        "##i",
        "##ṣ",
        "##ẹ́"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣiṣẹ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "ẹ",
        "́"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        " a",
        "iki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "yi",
        " aiki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "Ġa",
        "iki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "▁a",
        "iki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁y",
        "i",
        "▁a",
        "iki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "ai",
        "##ki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi aiki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁yi",
        "▁aiki"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "r",
        "ụ",
        "ọ",
        " ",
        "ọ",
        "r",
        "ụ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "rụ",
        "ọ",
        " ọrụ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "r",
        "á»¥",
        "á»į",
        "Ġ",
        "á»į",
        "r",
        "á»¥"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "r",
        "ụ",
        "ọ",
        "▁ọ",
        "r",
        "ụ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁r",
        "ụ",
        "ọ",
        "▁",
        "ọ",
        "r",
        "ụ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "r",
        "##ụ",
        "##ọ",
        "ọ",
        "##r",
        "##ụ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "rụọ ọrụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁r",
        "ụ",
        "ọ",
        "▁",
        "ọ",
        "r",
        "ụ"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 21,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "work",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁work"
      ],
      "english_equivalent": "to work",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġlearn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to learn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "k",
        "ọ",
        "́",
        " ",
        "�",
        "�",
        "̀",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "kọ",
        "́",
        " ẹ",
        "̀",
        "kọ",
        "́"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "k",
        "á»į",
        "Ìģ",
        "Ġ",
        "áº¹",
        "ÌĢ",
        "k",
        "á»į",
        "Ìģ"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "k",
        "ọ",
        "́",
        "▁",
        "ẹ",
        "̀",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁k",
        "ọ",
        "́",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "k",
        "##ọ́",
        "ẹ",
        "##̀",
        "##k",
        "##ọ́"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ́ ẹ̀kọ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁k",
        "ọ",
        "́",
        "▁",
        "ẹ",
        "̀",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "oy",
        "i"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "k",
        "oyi"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "oy",
        "i"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "k",
        "oyi"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁k",
        "oy",
        "i"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ko",
        "##yi"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "koyi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ko",
        "yi"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "m",
        "ụ",
        "ta",
        " i",
        "he"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "m",
        "ụta",
        " ihe"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "m",
        "á»¥",
        "ta",
        "Ġi",
        "he"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "m",
        "ụ",
        "ta",
        "▁ihe"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁m",
        "ụ",
        "ta",
        "▁i",
        "he"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "m",
        "##ụ",
        "##ta",
        "ih",
        "##e"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mụta ihe",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁m",
        "ụ",
        "ta",
        "▁i",
        "he"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 22,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "learn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁learn"
      ],
      "english_equivalent": "to learn",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "big",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ń",
        "l",
        "á"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ń",
        "lá"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ÅĦ",
        "lÃ¡"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ń",
        "lá"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ń",
        "l",
        "á"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ń",
        "##lá"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ńlá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ń",
        "lá"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "bab",
        "ba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "bab",
        "ba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "bab",
        "ba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "bab",
        "ba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁bab",
        "ba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ba",
        "##bba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "babba",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁bab",
        "ba"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "uk",
        "wu"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ukwu"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "uk",
        "wu"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "uk",
        "wu"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁u",
        "kw",
        "u"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "uk",
        "##wu"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ukwu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁uk",
        "wu"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 23,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "big",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁big"
      ],
      "english_equivalent": "big",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "beautiful",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁beautiful"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ar",
        "�",
        "�",
        "w",
        "à"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ar",
        "ẹ",
        "w",
        "à"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "ar",
        "áº¹",
        "w",
        "Ãł"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "ar",
        "ẹ",
        "w",
        "à"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁ar",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "w",
        "à"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ar",
        "##ẹ",
        "##w",
        "##à"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "arẹwà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁ar",
        "ẹ",
        "w",
        "à"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ky",
        "ak",
        "ky",
        "awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ky",
        "ak",
        "ky",
        "awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "ky",
        "ak",
        "ky",
        "awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "ky",
        "ak",
        "ky",
        "awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁k",
        "y",
        "ak",
        "ky",
        "awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ky",
        "##ak",
        "##ky",
        "##awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kyakkyawa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁ky",
        "ak",
        "ky",
        "awa"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "m",
        "ara",
        " m",
        "ma"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "m",
        "ara",
        " mma"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "m",
        "ara",
        "Ġm",
        "ma"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "mara",
        "▁mma"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁mar",
        "a",
        "▁m",
        "ma"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "mara",
        "mm",
        "##a"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mara mma",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁mara",
        "▁",
        "mma"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 24,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "fine",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁fine"
      ],
      "english_equivalent": "beautiful",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "good",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁good"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "d",
        "á",
        "ra"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "ára"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "d",
        "Ã¡",
        "ra"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "ára"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁d",
        "ára"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "dá",
        "##ra"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "dára",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁d",
        "ára"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "n",
        "ag",
        "ari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "nag",
        "ari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "n",
        "ag",
        "ari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "nag",
        "ari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁nag",
        "ari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "nag",
        "##ari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "nagari",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁na",
        "gari"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "á»į",
        "ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ọ",
        "ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ọ",
        "##ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọma",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ọ",
        "ma"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 25,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "correct",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁correct"
      ],
      "english_equivalent": "good",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "Ġhundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "▁hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one hundred",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "ọ",
        "g",
        "ọ",
        "́",
        "r",
        "ù",
        "n"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "ọ",
        "g",
        "ọ",
        "́",
        "r",
        "ùn"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "á»į",
        "g",
        "á»į",
        "Ìģ",
        "r",
        "Ã¹",
        "n"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "ọ",
        "g",
        "ọ",
        "́",
        "r",
        "ùn"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁",
        "ọ",
        "g",
        "ọ",
        "́",
        "r",
        "ù",
        "n"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "##g",
        "##ọ́",
        "##r",
        "##ùn"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọgọ́rùn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 8,
      "token_splits": [
        "▁",
        "ọ",
        "g",
        "ọ",
        "́",
        "r",
        "ù",
        "n"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "�",
        "�",
        "ari"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ɗ",
        "ari"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "É",
        "Ĺ",
        "ari"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ɗ",
        "ari"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "<0xC9>",
        "<0x97>",
        "ari"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "[UNK]"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ɗari",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ɗ",
        "ari"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "nar",
        "ị"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "nar",
        "ị"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "nar",
        "á»ĭ"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "nar",
        "ị"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁nar",
        "ị"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "nar",
        "##ị"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "narị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁na",
        "r",
        "ị"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "Ġhundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "▁hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 26,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one hundred",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁hundred"
      ],
      "english_equivalent": "one hundred",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "Ġthousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "▁thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "eng",
      "language_name": "English",
      "text": "one thousand",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "�",
        "�",
        "gb",
        "�",
        "�",
        "̀",
        "r",
        "ún"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ẹ",
        "gbẹ",
        "̀",
        "r",
        "ún"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "áº¹",
        "gb",
        "áº¹",
        "ÌĢ",
        "r",
        "Ãºn"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "ẹ",
        "gb",
        "ẹ",
        "̀",
        "r",
        "ún"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "gb",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ún"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.5
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "ẹ",
        "##g",
        "##b",
        "##ẹ̀",
        "##rú",
        "##n"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹgbẹ̀rún",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ẹ",
        "gb",
        "ẹ",
        "̀",
        "rún"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "d",
        "ub",
        "u"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "ubu"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "dub",
        "u"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "ubu"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁dub",
        "u"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "dub",
        "##u"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dubu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁du",
        "bu"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "uku"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "uku"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "uku"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "uku"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁p",
        "uk",
        "u"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "pu",
        "##ku"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "puku",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁pu",
        "ku"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "one",
        " thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "Ġthousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "▁thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "one",
        "thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 27,
      "category": "numbers",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "one thousand",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁one",
        "▁thousand"
      ],
      "english_equivalent": "one thousand",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Ol",
        "aw",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ola",
        "##wal",
        "##e"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Olawale",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Ola",
        "wale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "�",
        "�",
        "l",
        "á",
        "w",
        "ál",
        "é"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ọ",
        "lá",
        "w",
        "ál",
        "é"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "á»",
        "Į",
        "lÃ¡",
        "w",
        "Ã¡l",
        "Ã©"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ọ",
        "lá",
        "w",
        "á",
        "lé"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "l",
        "á",
        "w",
        "ál",
        "é"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ọ",
        "##lá",
        "##w",
        "##ál",
        "##é"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọláwálé",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "Ọ",
        "lá",
        "w",
        "ál",
        "é"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "ub",
        "ak",
        "ar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ab",
        "ub",
        "akar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "ub",
        "ak",
        "ar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "ubakar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ab",
        "ub",
        "ak",
        "ar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Abu",
        "##bak",
        "##ar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abubakar",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁Abubakar"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "nam",
        "di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "nam",
        "di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "nam",
        "di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "nam",
        "di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁N",
        "nam",
        "di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "##nam",
        "##di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnamdi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁N",
        "nam",
        "di"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "O",
        "law",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Ol",
        "aw",
        "ale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ola",
        "##wal",
        "##e"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 28,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Olawale",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Ola",
        "wale"
      ],
      "english_equivalent": "Olawale",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "de",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ade",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "de",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "A",
        "dey",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁A",
        "de",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ad",
        "##eye",
        "##mi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Adeyemi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Ade",
        "ye",
        "mi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "Ad",
        "é",
        "y",
        "�",
        "�",
        "m",
        "í"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ad",
        "é",
        "y",
        "ẹ",
        "mí"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Ad",
        "Ã©",
        "y",
        "áº¹",
        "mÃŃ"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ad",
        "é",
        "y",
        "ẹ",
        "mí"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁Ad",
        "é",
        "y",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "m",
        "í"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ad",
        "##é",
        "##y",
        "##ẹ",
        "##mí"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Adéyẹmí",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁A",
        "dé",
        "y",
        "ẹ",
        "mí"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "d",
        "ullah",
        "i"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "d",
        "ull",
        "ahi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "d",
        "ullah",
        "i"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Abd",
        "ulla",
        "hi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Abd",
        "ull",
        "ah",
        "i"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Abdullah",
        "##i"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abdullahi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Abdullah",
        "i"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ch",
        "in",
        "wend",
        "u"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ch",
        "in",
        "wend",
        "u"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ch",
        "in",
        "wend",
        "u"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Chin",
        "wend",
        "u"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ch",
        "in",
        "wend",
        "u"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Chin",
        "##wen",
        "##du"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chinwendu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Chin",
        "wend",
        "u"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "de",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ade",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "de",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "A",
        "dey",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁A",
        "de",
        "y",
        "emi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ad",
        "##eye",
        "##mi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 29,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Adeyemi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Ade",
        "ye",
        "mi"
      ],
      "english_equivalent": "Adeyemi",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "F",
        "ol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Fol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "F",
        "ol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Fo",
        "lake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Fol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "F",
        "##ola",
        "##ke"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Folake",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Fol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "F",
        "ọ",
        "l",
        "ák",
        "é"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "F",
        "ọ",
        "l",
        "ák",
        "é"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "F",
        "á»į",
        "l",
        "Ã¡k",
        "Ã©"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "F",
        "ọ",
        "lá",
        "ké"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁F",
        "ọ",
        "l",
        "ák",
        "é"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "F",
        "##ọ",
        "##lá",
        "##ké"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Fọláké",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁F",
        "ọ",
        "lá",
        "ké"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Hal",
        "ima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Hal",
        "ima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Hal",
        "ima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "H",
        "alima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Hal",
        "ima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Hal",
        "##ima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Halima",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Hal",
        "ima"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "�",
        "�",
        "z",
        "ọ",
        "ch",
        "uk",
        "wu"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ụ",
        "zọ",
        "ch",
        "ukwu"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "á»",
        "¤",
        "z",
        "á»į",
        "ch",
        "uk",
        "wu"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Ụ",
        "z",
        "ọ",
        "chukwu"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0xA4>",
        "z",
        "ọ",
        "ch",
        "uk",
        "w",
        "u"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "[UNK]"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụzọchukwu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁",
        "Ụ",
        "z",
        "ọc",
        "hu",
        "kw",
        "u"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "F",
        "ol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Fol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "F",
        "ol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Fo",
        "lake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Fol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "F",
        "##ola",
        "##ke"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 30,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Folake",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Fol",
        "ake"
      ],
      "english_equivalent": "Folake",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "##ode",
        "##le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Ayodele",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Ayo",
        "dele"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ay",
        "ọ",
        "̀",
        "d",
        "élé"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ay",
        "ọ",
        "̀",
        "d",
        "élé"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Ay",
        "á»į",
        "ÌĢ",
        "d",
        "Ã©lÃ©"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ay",
        "ọ",
        "̀",
        "dé",
        "lé"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Ay",
        "ọ",
        "̀",
        "d",
        "élé"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ay",
        "##ọ̀",
        "##dé",
        "##lé"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayọ̀délé",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Ay",
        "ọ",
        "̀",
        "d",
        "élé"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "I",
        "brahim"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "I",
        "brahim"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "I",
        "brahim"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Ibrahim"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁I",
        "bra",
        "him"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "Ibrahim"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ibrahim",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁Ibrahim"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ob",
        "i",
        "ọ",
        "ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ob",
        "i",
        "ọ",
        "ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ob",
        "i",
        "á»į",
        "ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Obi",
        "ọ",
        "ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ob",
        "i",
        "ọ",
        "ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ob",
        "##i",
        "##ọ",
        "##ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Obiọma",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Obi",
        "ọ",
        "ma"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Ay",
        "ode",
        "le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ay",
        "##ode",
        "##le"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 31,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ayodele",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Ayo",
        "dele"
      ],
      "english_equivalent": "Ayodele",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Lag",
        "os"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Lagos",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Lago",
        "s"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "�",
        "�",
        "k",
        "ó"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "È",
        "k",
        "ó"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ÃĪ",
        "k",
        "Ã³"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "È",
        "kó"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁È",
        "k",
        "ó"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "È",
        "##kó"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Èkó",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁È",
        "kó"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Lag",
        "os"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lagos",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Lago",
        "s"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Lag",
        "os"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Lagos",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Lago",
        "s"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "L",
        "agos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Lag",
        "os"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "Lagos"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 32,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Lago",
        "s"
      ],
      "english_equivalent": "Lagos",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Os",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Os",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Os",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Os",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Os",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Os",
        "##og",
        "##bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Osogbo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Os",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "�",
        "�",
        "̀",
        "�",
        "�",
        "�",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ọ",
        "̀",
        "ṣ",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "á»",
        "Į",
        "ÌĢ",
        "á¹",
        "£",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ọ",
        "̀",
        "ṣ",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "̀",
        "ṣ",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ọ",
        "##̀",
        "##ṣ",
        "##og",
        "##bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọ̀ṣogbo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "Ọ",
        "̀",
        "ṣ",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "M",
        "aid",
        "ug",
        "uri"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "M",
        "aid",
        "ug",
        "uri"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "M",
        "aid",
        "ug",
        "uri"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Maid",
        "ug",
        "uri"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁M",
        "aid",
        "ug",
        "uri"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Mai",
        "##du",
        "##gur",
        "##i"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Maiduguri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Mai",
        "du",
        "guri"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "On",
        "it",
        "sha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "On",
        "it",
        "sha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "On",
        "it",
        "sha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "On",
        "its",
        "ha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁On",
        "it",
        "sha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Oni",
        "##ts",
        "##ha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onitsha",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁On",
        "it",
        "sha"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "O",
        "sh",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "O",
        "sh",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "O",
        "sh",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "O",
        "sho",
        "g",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁O",
        "sh",
        "og",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Os",
        "##hog",
        "##bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 33,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oshogbo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Os",
        "ho",
        "g",
        "bo"
      ],
      "english_equivalent": "Osogbo",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "I",
        "bad",
        "an"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ib",
        "adan"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "I",
        "bad",
        "an"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "I",
        "badan"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁I",
        "bad",
        "an"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "I",
        "##bada",
        "##n"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Ibadan",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁I",
        "badan"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "�",
        "�",
        "b",
        "à",
        "d",
        "àn"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ì",
        "b",
        "à",
        "d",
        "àn"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "ÃĮ",
        "b",
        "Ãł",
        "d",
        "Ãłn"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ì",
        "b",
        "à",
        "d",
        "àn"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁",
        "Ì",
        "b",
        "à",
        "d",
        "à",
        "n"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ì",
        "##bà",
        "##dà",
        "##n"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìbàdàn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "Ì",
        "b",
        "à",
        "d",
        "àn"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "K",
        "ano"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "K",
        "ano"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "K",
        "ano"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "K",
        "ano"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁K",
        "ano"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Kan",
        "##o"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kano",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁Kano"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "new",
        "i"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "new",
        "i"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "new",
        "i"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "new",
        "i"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁N",
        "new",
        "i"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "N",
        "##ne",
        "##wi"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nnewi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁N",
        "ne",
        "wi"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "I",
        "bad",
        "an"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ib",
        "adan"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "I",
        "bad",
        "an"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "I",
        "badan"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁I",
        "bad",
        "an"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "I",
        "##bada",
        "##n"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 34,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Ibadan",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁I",
        "badan"
      ],
      "english_equivalent": "Ibadan",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Abe",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Abe",
        "##oku",
        "##ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abeokuta",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Abe",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "Ab",
        "�",
        "�",
        "́",
        "ò",
        "k",
        "ú",
        "ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "Ab",
        "ẹ",
        "́",
        "ò",
        "k",
        "ú",
        "ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "Ab",
        "áº¹",
        "Ìģ",
        "Ã²",
        "k",
        "Ãº",
        "ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Ab",
        "ẹ",
        "́",
        "ò",
        "kú",
        "ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁Ab",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "ò",
        "k",
        "ú",
        "ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ab",
        "##ẹ́",
        "##ò",
        "##kú",
        "##ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abẹ́òkúta",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁Ab",
        "ẹ",
        "́",
        "ò",
        "kú",
        "ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "K",
        "ad",
        "una"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Kad",
        "una"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "K",
        "ad",
        "una"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Kad",
        "una"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁K",
        "ad",
        "una"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Kad",
        "##una"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kaduna",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁Kaduna"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "En",
        "ug",
        "wu"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "En",
        "ug",
        "wu"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "En",
        "ug",
        "wu"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "En",
        "ug",
        "wu"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁En",
        "ug",
        "w",
        "u"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "En",
        "##ug",
        "##wu"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Enugwu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁En",
        "ug",
        "wu"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Abe",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁A",
        "be",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Abe",
        "##oku",
        "##ta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 35,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeokuta",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Abe",
        "ok",
        "uta"
      ],
      "english_equivalent": "Abeokuta",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "The",
        " food",
        " is",
        " delicious"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "The",
        " food",
        " is",
        " delicious"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "The",
        "Ġfood",
        "Ġis",
        "Ġdelicious"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "The",
        "▁food",
        "▁is",
        "▁delicious"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁The",
        "▁food",
        "▁is",
        "▁delicious"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "The",
        "food",
        "is",
        "del",
        "##icio",
        "##us"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "The food is delicious",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁The",
        "▁food",
        "▁is",
        "▁de",
        "li",
        "cious"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "O",
        "ún",
        "j",
        "�",
        "�",
        " n",
        "á",
        "à",
        " d",
        "ù",
        "n",
        " g",
        "idi",
        " gan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "O",
        "ún",
        "jẹ",
        " náà",
        " dù",
        "n",
        " g",
        "idi",
        " gan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "O",
        "Ãºn",
        "j",
        "áº¹",
        "ĠnÃ¡",
        "Ãł",
        "ĠdÃ¹",
        "n",
        "Ġg",
        "idi",
        "Ġgan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "O",
        "ún",
        "j",
        "ẹ",
        "▁ná",
        "à",
        "▁dù",
        "n",
        "▁gidi",
        "▁gan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁O",
        "ún",
        "j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "▁ná",
        "à",
        "▁d",
        "ù",
        "n",
        "▁g",
        "idi",
        "▁gan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "O",
        "##ún",
        "##j",
        "##ẹ",
        "náà",
        "dù",
        "##n",
        "gi",
        "##di",
        "gan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Oúnjẹ náà dùn gidi gan",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁O",
        "ún",
        "j",
        "ẹ",
        "▁ná",
        "à",
        "▁dù",
        "n",
        "▁gid",
        "i",
        "▁gan"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.833
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Ab",
        "inc",
        "in",
        " nan",
        " y",
        "ana",
        " da",
        " da",
        "�",
        "�",
        "i"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Ab",
        "inc",
        "in",
        " nan",
        " yana",
        " da",
        " da",
        "ɗ",
        "i"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "Ab",
        "inc",
        "in",
        "Ġnan",
        "Ġyana",
        "Ġda",
        "Ġda",
        "É",
        "Ĺi"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "Ab",
        "in",
        "cin",
        "▁nan",
        "▁yana",
        "▁da",
        "▁da",
        "ɗ",
        "i"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Ab",
        "inc",
        "in",
        "▁nan",
        "▁y",
        "ana",
        "▁da",
        "▁da",
        "<0xC9>",
        "<0x97>",
        "i"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "Abi",
        "##nci",
        "##n",
        "nan",
        "yana",
        "da",
        "[UNK]"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abincin nan yana da daɗi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 8,
      "token_splits": [
        "▁Abi",
        "ncin",
        "▁nan",
        "▁yana",
        "▁da",
        "▁da",
        "ɗ",
        "i"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "N",
        "ri",
        " a",
        " d",
        "ị",
        " ",
        "ụ",
        "t",
        "ọ",
        " n",
        "ke",
        " uk",
        "w",
        "uu"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "N",
        "ri",
        " a",
        " dị",
        " ụtọ",
        " nke",
        " ukwuu"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "N",
        "ri",
        "Ġa",
        "Ġdá»ĭ",
        "Ġ",
        "á»¥",
        "t",
        "á»į",
        "Ġn",
        "ke",
        "Ġuk",
        "w",
        "uu"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "N",
        "ri",
        "▁a",
        "▁dị",
        "▁",
        "ụ",
        "t",
        "ọ",
        "▁nke",
        "▁uk",
        "w",
        "uu"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁N",
        "ri",
        "▁a",
        "▁d",
        "ị",
        "▁",
        "ụ",
        "t",
        "ọ",
        "▁n",
        "ke",
        "▁u",
        "kw",
        "uu"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "Nr",
        "##i",
        "a",
        "d",
        "##ị",
        "ụ",
        "##t",
        "##ọ",
        "n",
        "##ke",
        "uk",
        "##wu",
        "##u"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nri a dị ụtọ nke ukwuu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁Nr",
        "i",
        "▁a",
        "▁dị",
        "▁",
        "ụ",
        "t",
        "ọ",
        "▁",
        "nke",
        "▁uk",
        "wu",
        "u"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Dis",
        " food",
        " sweet",
        " well",
        " well"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Dis",
        " food",
        " sweet",
        " well",
        " well"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Dis",
        "Ġfood",
        "Ġsweet",
        "Ġwell",
        "Ġwell"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Dis",
        "▁food",
        "▁sweet",
        "▁well",
        "▁well"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Dis",
        "▁food",
        "▁sweet",
        "▁well",
        "▁well"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Dis",
        "food",
        "sweet",
        "well",
        "well"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 36,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis food sweet well well",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Dis",
        "▁food",
        "▁sweet",
        "▁well",
        "▁well"
      ],
      "english_equivalent": "The food is delicious",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "I",
        " am",
        " going",
        " to",
        " the",
        " market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "I",
        " am",
        " going",
        " to",
        " the",
        " market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "I",
        "Ġam",
        "Ġgoing",
        "Ġto",
        "Ġthe",
        "Ġmarket"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "I",
        "▁am",
        "▁going",
        "▁to",
        "▁the",
        "▁market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁I",
        "▁am",
        "▁going",
        "▁to",
        "▁the",
        "▁market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "I",
        "am",
        "going",
        "to",
        "the",
        "market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "I am going to the market",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁I",
        "▁am",
        "▁going",
        "▁to",
        "▁the",
        "▁market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Mo",
        " ",
        "ń",
        " l",
        "ọ",
        " sí",
        " ",
        "ọ",
        "jà"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Mo",
        " ń",
        " lọ",
        " sí",
        " ọ",
        "jà"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "Mo",
        "Ġ",
        "ÅĦ",
        "Ġl",
        "á»į",
        "ĠsÃŃ",
        "Ġ",
        "á»į",
        "jÃł"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 7,
      "token_splits": [
        "Mo",
        "▁",
        "ń",
        "▁lọ",
        "▁sí",
        "▁ọ",
        "jà"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁Mo",
        "▁",
        "ń",
        "▁l",
        "ọ",
        "▁sí",
        "▁",
        "ọ",
        "jà"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "Mo",
        "ń",
        "l",
        "##ọ",
        "sí",
        "ọ",
        "##jà"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Mo ń lọ sí ọjà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁Mo",
        "▁",
        "ń",
        "▁l",
        "ọ",
        "▁sí",
        "▁",
        "ọ",
        "j",
        "à"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "In",
        "a",
        " zu",
        "wa",
        " kas",
        "u",
        "wa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ina",
        " zuwa",
        " kas",
        "uwa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "In",
        "a",
        "Ġzu",
        "wa",
        "Ġkas",
        "u",
        "wa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ina",
        "▁zu",
        "wa",
        "▁kas",
        "uwa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁In",
        "a",
        "▁zu",
        "wa",
        "▁k",
        "as",
        "u",
        "wa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ina",
        "zu",
        "##wa",
        "kas",
        "##uwa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina zuwa kasuwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Ina",
        "▁zuwa",
        "▁kasuwa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "Ana",
        " m",
        " ag",
        "a",
        " ah",
        "ị",
        "a"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ana",
        " m",
        " aga",
        " ahịa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "Ana",
        "Ġm",
        "Ġag",
        "a",
        "Ġah",
        "á»ĭa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Ana",
        "▁m",
        "▁aga",
        "▁ah",
        "ịa"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁Ana",
        "▁m",
        "▁ag",
        "a",
        "▁ah",
        "ị",
        "a"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Ana",
        "m",
        "aga",
        "ah",
        "##ị",
        "##a"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ana m aga ahịa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁Ana",
        "▁m",
        "▁aga",
        "▁ah",
        "ị",
        "a"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "I",
        " de",
        "y",
        " go",
        " market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "I",
        " dey",
        " go",
        " market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "I",
        "Ġde",
        "y",
        "Ġgo",
        "Ġmarket"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "I",
        "▁dey",
        "▁go",
        "▁market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁I",
        "▁de",
        "y",
        "▁go",
        "▁market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "I",
        "de",
        "##y",
        "go",
        "market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 37,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "I dey go market",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁I",
        "▁de",
        "y",
        "▁go",
        "▁market"
      ],
      "english_equivalent": "I am going to the market",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "How",
        " much",
        " does",
        " this",
        " cost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "How",
        " much",
        " does",
        " this",
        " cost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "How",
        "Ġmuch",
        "Ġdoes",
        "Ġthis",
        "Ġcost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "How",
        "▁much",
        "▁does",
        "▁this",
        "▁cost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁How",
        "▁much",
        "▁does",
        "▁this",
        "▁cost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "How",
        "much",
        "does",
        "this",
        "cost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "How much does this cost?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁How",
        "▁much",
        "▁does",
        "▁this",
        "▁cost",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "�",
        "�",
        "̀",
        "l",
        "ó",
        " ni",
        " è",
        "y",
        "í",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 8,
      "token_splits": [
        "Ẹ",
        "̀",
        "ló",
        " ni",
        " è",
        "y",
        "í",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "áº",
        "¸",
        "ÌĢ",
        "l",
        "Ã³",
        "Ġni",
        "ĠÃ¨",
        "y",
        "ÃŃ",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "Ẹ",
        "̀",
        "ló",
        "▁ni",
        "▁è",
        "y",
        "í",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "̀",
        "l",
        "ó",
        "▁ni",
        "▁è",
        "y",
        "í",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 8,
      "token_splits": [
        "Ẹ",
        "##̀",
        "##ló",
        "ni",
        "è",
        "##y",
        "##í",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀ló ni èyí?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "Ẹ",
        "̀",
        "ló",
        "▁ni",
        "▁è",
        "y",
        "í",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "N",
        "awa",
        " ne",
        " wann",
        "an",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "N",
        "awa",
        " ne",
        " wannan",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "N",
        "awa",
        "Ġne",
        "Ġwann",
        "an",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "N",
        "awa",
        "▁ne",
        "▁wann",
        "an",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁N",
        "awa",
        "▁ne",
        "▁w",
        "ann",
        "an",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Na",
        "##wa",
        "ne",
        "wa",
        "##nnan",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nawa ne wannan?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Na",
        "wa",
        "▁ne",
        "▁wannan",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "E",
        "go",
        " ole",
        " ka",
        " n",
        "ke",
        " a",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "E",
        "go",
        " ole",
        " ka",
        " nke",
        " a",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "E",
        "go",
        "Ġole",
        "Ġka",
        "Ġn",
        "ke",
        "Ġa",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Ego",
        "▁ole",
        "▁ka",
        "▁nke",
        "▁a",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁E",
        "go",
        "▁o",
        "le",
        "▁ka",
        "▁n",
        "ke",
        "▁a",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "Ego",
        "ole",
        "ka",
        "n",
        "##ke",
        "a",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ole ka nke a?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁Ego",
        "▁ole",
        "▁ka",
        "▁",
        "nke",
        "▁a",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "How",
        " much",
        " be",
        " dis",
        " one",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "How",
        " much",
        " be",
        " dis",
        " one",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "How",
        "Ġmuch",
        "Ġbe",
        "Ġdis",
        "Ġone",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "How",
        "▁much",
        "▁be",
        "▁dis",
        "▁one",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁How",
        "▁much",
        "▁be",
        "▁dis",
        "▁one",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "How",
        "much",
        "be",
        "dis",
        "one",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 38,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "How much be dis one?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁How",
        "▁much",
        "▁be",
        "▁dis",
        "▁one",
        "?"
      ],
      "english_equivalent": "How much does this cost?",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Please",
        " help",
        " me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Please",
        " help",
        " me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Please",
        "Ġhelp",
        "Ġme"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Please",
        "▁help",
        "▁me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Please",
        "▁help",
        "▁me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Please",
        "help",
        "me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Please help me",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Please",
        "▁help",
        "▁me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "�",
        "�",
        " j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        ",",
        " ",
        "�",
        "�",
        " r",
        "àn",
        " mí",
        " l",
        "ọ",
        "́",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Ẹ",
        " j",
        "ọ",
        "̀",
        "wọ",
        "́",
        ",",
        " ẹ",
        " r",
        "àn",
        " mí",
        " lọ",
        "́",
        "wọ",
        "́"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 20,
      "token_splits": [
        "áº",
        "¸",
        "Ġj",
        "á»į",
        "ÌĢ",
        "w",
        "á»į",
        "Ìģ",
        ",",
        "Ġ",
        "áº¹",
        "Ġr",
        "Ãłn",
        "ĠmÃŃ",
        "Ġl",
        "á»į",
        "Ìģ",
        "w",
        "á»į",
        "Ìģ"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 6.667
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "Ẹ",
        "▁j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        ",",
        "▁",
        "ẹ",
        "▁r",
        "àn",
        "▁mí",
        "▁lọ",
        "́",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "▁j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        ",",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "▁r",
        "à",
        "n",
        "▁m",
        "í",
        "▁l",
        "ọ",
        "́",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 8.667
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "Ẹ",
        "j",
        "##ọ̀",
        "##w",
        "##ọ́",
        ",",
        "ẹ",
        "r",
        "##àn",
        "mí",
        "l",
        "##ọ́",
        "##w",
        "##ọ́"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 4.667
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́, ẹ ràn mí lọ́wọ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 21,
      "token_splits": [
        "▁",
        "Ẹ",
        "▁j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        ",",
        "▁",
        "ẹ",
        "▁",
        "rà",
        "n",
        "▁mí",
        "▁l",
        "ọ",
        "́",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "Don",
        " Allah",
        " ka",
        " t",
        "aim",
        "ake",
        " ni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Don",
        " Allah",
        " ka",
        " taim",
        "ake",
        " ni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "Don",
        "ĠAllah",
        "Ġka",
        "Ġt",
        "aim",
        "ake",
        "Ġni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Don",
        "▁Allah",
        "▁ka",
        "▁tai",
        "make",
        "▁ni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁Don",
        "▁All",
        "ah",
        "▁ka",
        "▁t",
        "aim",
        "ake",
        "▁ni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "Don",
        "Allah",
        "ka",
        "tai",
        "##mak",
        "##e",
        "ni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah ka taimake ni",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁Don",
        "▁Allah",
        "▁ka",
        "▁tai",
        "make",
        "▁ni"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "B",
        "iko",
        " ny",
        "ere",
        " m",
        " aka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "B",
        "iko",
        " nyere",
        " m",
        " aka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "B",
        "iko",
        "Ġny",
        "ere",
        "Ġm",
        "Ġaka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "B",
        "iko",
        "▁ny",
        "ere",
        "▁m",
        "▁aka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁B",
        "iko",
        "▁ny",
        "ere",
        "▁m",
        "▁a",
        "ka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Bi",
        "##ko",
        "nye",
        "##re",
        "m",
        "aka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko nyere m aka",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Bi",
        "ko",
        "▁nyere",
        "▁m",
        "▁aka"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "eg",
        " help",
        " me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "eg",
        " help",
        " me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "eg",
        "Ġhelp",
        "Ġme"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "A",
        "beg",
        "▁help",
        "▁me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ab",
        "eg",
        "▁help",
        "▁me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Abe",
        "##g",
        "help",
        "me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 39,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg help me",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁A",
        "beg",
        "▁help",
        "▁me"
      ],
      "english_equivalent": "Please help me",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        " is",
        " the",
        " hospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        " is",
        " the",
        " hospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        "Ġis",
        "Ġthe",
        "Ġhospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        "▁is",
        "▁the",
        "▁hospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Where",
        "▁is",
        "▁the",
        "▁hospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        "is",
        "the",
        "hospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "eng",
      "language_name": "English",
      "text": "Where is the hospital?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Where",
        "▁is",
        "▁the",
        "▁hospital",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "N",
        "í",
        "bo",
        " ni",
        " il",
        "é",
        " �",
        "�",
        "w",
        "ò",
        "s",
        "àn",
        " w",
        "à",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "Ní",
        "bo",
        " ni",
        " il",
        "é",
        " ì",
        "w",
        "òs",
        "àn",
        " w",
        "à",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "N",
        "ÃŃ",
        "bo",
        "Ġni",
        "Ġil",
        "Ã©",
        "ĠÃ",
        "¬",
        "w",
        "Ã²",
        "s",
        "Ãłn",
        "Ġw",
        "Ãł",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "N",
        "í",
        "bo",
        "▁ni",
        "▁il",
        "é",
        "▁ì",
        "w",
        "òs",
        "àn",
        "▁w",
        "à",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.6
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 16,
      "token_splits": [
        "▁N",
        "í",
        "bo",
        "▁ni",
        "▁il",
        "é",
        "▁",
        "ì",
        "w",
        "ò",
        "s",
        "à",
        "n",
        "▁w",
        "à",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 3.2
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "Ní",
        "##bo",
        "ni",
        "il",
        "##é",
        "ì",
        "##w",
        "##òs",
        "##àn",
        "w",
        "##à",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Níbo ni ilé ìwòsàn wà?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁Ní",
        "bo",
        "▁ni",
        "▁i",
        "lé",
        "▁",
        "ì",
        "w",
        "ò",
        "s",
        "àn",
        "▁w",
        "à",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.8
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "In",
        "a",
        " as",
        "ib",
        "iti",
        " y",
        "ake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Ina",
        " as",
        "ib",
        "iti",
        " yake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "In",
        "a",
        "Ġas",
        "ib",
        "iti",
        "Ġy",
        "ake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Ina",
        "▁a",
        "sib",
        "iti",
        "▁yake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁In",
        "a",
        "▁as",
        "ib",
        "iti",
        "▁y",
        "ake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Ina",
        "asi",
        "##biti",
        "yake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ina asibiti yake?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Ina",
        "▁asi",
        "biti",
        "▁yake",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "O",
        "lee",
        " e",
        "be",
        " ",
        "ụ",
        "l",
        "ọ",
        " ",
        "ọ",
        "gw",
        "ụ",
        " d",
        "ị",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "O",
        "lee",
        " ebe",
        " ụlọ",
        " ọgwụ",
        " dị",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "O",
        "lee",
        "Ġe",
        "be",
        "Ġ",
        "á»¥",
        "l",
        "á»į",
        "Ġ",
        "á»į",
        "gw",
        "á»¥",
        "Ġdá»ĭ",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.8
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "O",
        "lee",
        "▁e",
        "be",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ọ",
        "gw",
        "ụ",
        "▁dị",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.6
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 16,
      "token_splits": [
        "▁O",
        "lee",
        "▁e",
        "be",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁",
        "ọ",
        "g",
        "w",
        "ụ",
        "▁d",
        "ị",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 3.2
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "Ole",
        "##e",
        "e",
        "##be",
        "ụ",
        "##l",
        "##ọ",
        "ọ",
        "##g",
        "##w",
        "##ụ",
        "d",
        "##ị",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.8
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Olee ebe ụlọ ọgwụ dị?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁Ole",
        "e",
        "▁e",
        "be",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁",
        "ọ",
        "g",
        "w",
        "ụ",
        "▁dị",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        " hospital",
        " de",
        "y",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Where",
        " hospital",
        " dey",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.8
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        "Ġhospital",
        "Ġde",
        "y",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Where",
        "▁hospital",
        "▁dey",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.8
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁Where",
        "▁hospital",
        "▁de",
        "y",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Where",
        "hospital",
        "de",
        "##y",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 40,
      "category": "phrases",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Where hospital dey?",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Where",
        "▁hospital",
        "▁de",
        "y",
        "?"
      ],
      "english_equivalent": "Where is the hospital?",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "The",
        " children",
        " are",
        " playing",
        " in",
        " the",
        " compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "The",
        " children",
        " are",
        " playing",
        " in",
        " the",
        " compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "The",
        "Ġchildren",
        "Ġare",
        "Ġplaying",
        "Ġin",
        "Ġthe",
        "Ġcompound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 7,
      "token_splits": [
        "The",
        "▁children",
        "▁are",
        "▁playing",
        "▁in",
        "▁the",
        "▁compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁The",
        "▁children",
        "▁are",
        "▁playing",
        "▁in",
        "▁the",
        "▁compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "The",
        "children",
        "are",
        "playing",
        "in",
        "the",
        "compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The children are playing in the compound",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁The",
        "▁children",
        "▁are",
        "▁playing",
        "▁in",
        "▁the",
        "▁compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "À",
        "w",
        "ọn",
        " ",
        "ọ",
        "m",
        "ọ",
        " ",
        "ń",
        " �",
        "�",
        "�",
        "er",
        "é",
        " n",
        "ín",
        "ú",
        " à",
        "gb",
        "à",
        "l",
        "á"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 14,
      "token_splits": [
        "À",
        "wọn",
        " ọmọ",
        " ń",
        " ṣ",
        "er",
        "é",
        " n",
        "ín",
        "ú",
        " à",
        "gb",
        "à",
        "lá"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 21,
      "token_splits": [
        "ÃĢ",
        "w",
        "á»įn",
        "Ġ",
        "á»į",
        "m",
        "á»į",
        "Ġ",
        "ÅĦ",
        "Ġá",
        "¹",
        "£",
        "er",
        "Ã©",
        "Ġn",
        "ÃŃn",
        "Ãº",
        "ĠÃł",
        "gb",
        "Ãł",
        "lÃ¡"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 19,
      "token_splits": [
        "À",
        "w",
        "ọn",
        "▁ọ",
        "m",
        "ọ",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "er",
        "é",
        "▁n",
        "ín",
        "ú",
        "▁à",
        "gb",
        "à",
        "lá"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.714
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁À",
        "w",
        "ọ",
        "n",
        "▁",
        "ọ",
        "m",
        "ọ",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "er",
        "é",
        "▁n",
        "ín",
        "ú",
        "▁à",
        "gb",
        "à",
        "l",
        "á"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 16,
      "token_splits": [
        "À",
        "##w",
        "##ọn",
        "ọ",
        "##m",
        "##ọ",
        "ń",
        "ṣ",
        "##er",
        "##é",
        "ní",
        "##nú",
        "à",
        "##g",
        "##bà",
        "##lá"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.286
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọmọ ń ṣeré nínú àgbàlá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 20,
      "token_splits": [
        "▁À",
        "w",
        "ọ",
        "n",
        "▁",
        "ọ",
        "m",
        "ọ",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "er",
        "é",
        "▁ní",
        "nú",
        "▁à",
        "gb",
        "à",
        "lá"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.857
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "Y",
        "ara",
        " sun",
        "a",
        " was",
        "a",
        " a",
        " c",
        "ik",
        "in",
        " fil",
        "i"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.714
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "Y",
        "ara",
        " suna",
        " wasa",
        " a",
        " cikin",
        " fili"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "Y",
        "ara",
        "Ġsun",
        "a",
        "Ġwas",
        "a",
        "Ġa",
        "Ġc",
        "ik",
        "in",
        "Ġfil",
        "i"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.714
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "Y",
        "ara",
        "▁suna",
        "▁was",
        "a",
        "▁a",
        "▁cikin",
        "▁fili"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁Y",
        "ara",
        "▁sun",
        "a",
        "▁was",
        "a",
        "▁a",
        "▁c",
        "ik",
        "in",
        "▁fil",
        "i"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.714
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Ya",
        "##ra",
        "sun",
        "##a",
        "was",
        "##a",
        "a",
        "ci",
        "##kin",
        "fil",
        "##i"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yara suna wasa a cikin fili",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁Yara",
        "▁suna",
        "▁was",
        "a",
        "▁a",
        "▁cikin",
        "▁fili"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "�",
        "�",
        "m",
        "ụ",
        "aka",
        " na",
        "-",
        "eg",
        "wu",
        " eg",
        "wu",
        " n",
        "'",
        "og",
        "ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Ụ",
        "mụ",
        "aka",
        " na",
        "-eg",
        "wu",
        " egwu",
        " n",
        "'",
        "og",
        "ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "á»",
        "¤",
        "m",
        "á»¥",
        "aka",
        "Ġna",
        "-",
        "eg",
        "wu",
        "Ġeg",
        "wu",
        "Ġn",
        "'",
        "og",
        "ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 14,
      "token_splits": [
        "Ụ",
        "m",
        "ụ",
        "aka",
        "▁na",
        "-",
        "eg",
        "wu",
        "▁eg",
        "wu",
        "▁n",
        "'",
        "og",
        "ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 19,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0xA4>",
        "m",
        "ụ",
        "aka",
        "▁na",
        "-",
        "eg",
        "w",
        "u",
        "▁eg",
        "w",
        "u",
        "▁n",
        "'",
        "og",
        "ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.714
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "[UNK]",
        "na",
        "-",
        "e",
        "##g",
        "##wu",
        "e",
        "##g",
        "##wu",
        "n",
        "'",
        "og",
        "##ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.857
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụaka na-egwu egwu n'ogige",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁",
        "Ụ",
        "m",
        "ụ",
        "aka",
        "▁na",
        "-",
        "eg",
        "wu",
        "▁eg",
        "wu",
        "▁n",
        "'",
        "og",
        "ige"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "De",
        " pik",
        "in",
        " dem",
        " de",
        "y",
        " play",
        " for",
        " compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 8,
      "token_splits": [
        "De",
        " p",
        "ikin",
        " dem",
        " dey",
        " play",
        " for",
        " compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "De",
        "Ġpik",
        "in",
        "Ġdem",
        "Ġde",
        "y",
        "Ġplay",
        "Ġfor",
        "Ġcompound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "De",
        "▁pik",
        "in",
        "▁dem",
        "▁dey",
        "▁play",
        "▁for",
        "▁compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.143
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁De",
        "▁p",
        "ik",
        "in",
        "▁dem",
        "▁de",
        "y",
        "▁play",
        "▁for",
        "▁compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.429
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "De",
        "pi",
        "##kin",
        "dem",
        "de",
        "##y",
        "play",
        "for",
        "compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 41,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De pikin dem dey play for compound",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁De",
        "▁pik",
        "in",
        "▁dem",
        "▁de",
        "y",
        "▁play",
        "▁for",
        "▁compound"
      ],
      "english_equivalent": "The children are playing in the compound",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "My",
        " grandmother",
        " told",
        " us",
        " stories",
        " every",
        " night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "My",
        " grandmother",
        " told",
        " us",
        " stories",
        " every",
        " night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "My",
        "Ġgrandmother",
        "Ġtold",
        "Ġus",
        "Ġstories",
        "Ġevery",
        "Ġnight"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 7,
      "token_splits": [
        "My",
        "▁grandmother",
        "▁told",
        "▁us",
        "▁stories",
        "▁every",
        "▁night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁My",
        "▁grandmother",
        "▁told",
        "▁us",
        "▁stories",
        "▁every",
        "▁night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "My",
        "grandmother",
        "told",
        "us",
        "stories",
        "every",
        "night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother told us stories every night",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁My",
        "▁grand",
        "mo",
        "ther",
        "▁told",
        "▁us",
        "▁stories",
        "▁every",
        "▁night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 30,
      "token_splits": [
        "�",
        "�",
        "y",
        "á",
        " à",
        "gb",
        "à",
        " mi",
        " má",
        "a",
        " ",
        "ń",
        " s",
        "ọ",
        " �",
        "�",
        "t",
        "àn",
        " f",
        "ún",
        " wa",
        " n",
        "í",
        " al",
        "�",
        "�",
        "́",
        " gb",
        "og",
        "bo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 4.286
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 21,
      "token_splits": [
        "Ì",
        "y",
        "á",
        " à",
        "gb",
        "à",
        " mi",
        " má",
        "a",
        " ń",
        " sọ",
        " ì",
        "t",
        "àn",
        " fún",
        " wa",
        " ní",
        " al",
        "ẹ",
        "́",
        " gbogbo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 27,
      "token_splits": [
        "ÃĮ",
        "y",
        "Ã¡",
        "ĠÃł",
        "gb",
        "Ãł",
        "Ġmi",
        "ĠmÃ¡",
        "a",
        "Ġ",
        "ÅĦ",
        "Ġs",
        "á»į",
        "ĠÃ",
        "¬",
        "t",
        "Ãłn",
        "Ġf",
        "Ãºn",
        "Ġwa",
        "ĠnÃŃ",
        "Ġal",
        "áº¹",
        "Ìģ",
        "Ġgb",
        "og",
        "bo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.857
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 26,
      "token_splits": [
        "Ì",
        "y",
        "á",
        "▁à",
        "gb",
        "à",
        "▁mi",
        "▁má",
        "a",
        "▁",
        "ń",
        "▁s",
        "ọ",
        "▁ì",
        "t",
        "àn",
        "▁f",
        "ún",
        "▁wa",
        "▁ní",
        "▁al",
        "ẹ",
        "́",
        "▁gb",
        "og",
        "bo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.714
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 33,
      "token_splits": [
        "▁",
        "Ì",
        "y",
        "á",
        "▁à",
        "gb",
        "à",
        "▁mi",
        "▁má",
        "a",
        "▁",
        "ń",
        "▁s",
        "ọ",
        "▁",
        "ì",
        "t",
        "à",
        "n",
        "▁f",
        "ún",
        "▁wa",
        "▁n",
        "í",
        "▁al",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁g",
        "b",
        "og",
        "bo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 4.714
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 24,
      "token_splits": [
        "Ì",
        "##y",
        "##á",
        "à",
        "##g",
        "##bà",
        "mi",
        "má",
        "##a",
        "ń",
        "s",
        "##ọ",
        "ì",
        "##tà",
        "##n",
        "f",
        "##ún",
        "wa",
        "ní",
        "al",
        "##ẹ́",
        "g",
        "##bo",
        "##gbo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.429
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi máa ń sọ ìtàn fún wa ní alẹ́ gbogbo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 28,
      "token_splits": [
        "▁",
        "Ì",
        "y",
        "á",
        "▁à",
        "gb",
        "à",
        "▁mi",
        "▁má",
        "a",
        "▁",
        "ń",
        "▁s",
        "ọ",
        "▁",
        "ì",
        "tà",
        "n",
        "▁fú",
        "n",
        "▁wa",
        "▁ní",
        "▁al",
        "ẹ",
        "́",
        "▁g",
        "bog",
        "bo"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.111
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "K",
        "akk",
        "ata",
        " t",
        "ana",
        " ba",
        " mu",
        " lab",
        "ari",
        " k",
        "ow",
        "ace",
        " dare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.857
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "K",
        "akk",
        "ata",
        " tana",
        " ba",
        " mu",
        " lab",
        "ari",
        " kow",
        "ace",
        " dare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "K",
        "akk",
        "ata",
        "Ġt",
        "ana",
        "Ġba",
        "Ġmu",
        "Ġlab",
        "ari",
        "Ġk",
        "ow",
        "ace",
        "Ġdare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.857
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "Kak",
        "kata",
        "▁tana",
        "▁ba",
        "▁mu",
        "▁lab",
        "ari",
        "▁kow",
        "ace",
        "▁dare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.429
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁K",
        "ak",
        "k",
        "ata",
        "▁t",
        "ana",
        "▁ba",
        "▁mu",
        "▁lab",
        "ari",
        "▁k",
        "ow",
        "ace",
        "▁dare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "Ka",
        "##kka",
        "##ta",
        "tan",
        "##a",
        "ba",
        "mu",
        "laba",
        "##ri",
        "ko",
        "##wa",
        "##ce",
        "dare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.857
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tana ba mu labari kowace dare",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Kak",
        "kata",
        "▁tana",
        "▁ba",
        "▁mu",
        "▁la",
        "bari",
        "▁",
        "kowa",
        "ce",
        "▁dare"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "N",
        "ne",
        " n",
        "ne",
        " m",
        " na",
        "-",
        "ak",
        "ọ",
        "r",
        "ọ",
        " any",
        "ị",
        " ak",
        "ụ",
        "k",
        "ọ",
        " k",
        "wa",
        " ab",
        "al",
        "ị"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.143
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "N",
        "ne",
        " nne",
        " m",
        " na",
        "-ak",
        "ọrọ",
        " anyị",
        " akụkọ",
        " kwa",
        " abal",
        "ị"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.714
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 22,
      "token_splits": [
        "N",
        "ne",
        "Ġn",
        "ne",
        "Ġm",
        "Ġna",
        "-",
        "ak",
        "á»į",
        "r",
        "á»į",
        "Ġany",
        "á»ĭ",
        "Ġak",
        "á»¥",
        "k",
        "á»į",
        "Ġk",
        "wa",
        "Ġab",
        "al",
        "á»ĭ"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.143
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "ne",
        "▁n",
        "ne",
        "▁m",
        "▁na",
        "-",
        "ak",
        "ọ",
        "r",
        "ọ",
        "▁any",
        "ị",
        "▁ak",
        "ụ",
        "k",
        "ọ",
        "▁kwa",
        "▁ab",
        "al",
        "ị"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁N",
        "ne",
        "▁n",
        "ne",
        "▁m",
        "▁na",
        "-",
        "ak",
        "ọ",
        "r",
        "ọ",
        "▁any",
        "ị",
        "▁ak",
        "ụ",
        "k",
        "ọ",
        "▁k",
        "wa",
        "▁ab",
        "al",
        "ị"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.143
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "##ne",
        "nn",
        "##e",
        "m",
        "na",
        "-",
        "ak",
        "##ọ",
        "##r",
        "##ọ",
        "any",
        "##ị",
        "ak",
        "##ụ",
        "##k",
        "##ọ",
        "kwa",
        "ab",
        "##al",
        "##ị"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-akọrọ anyị akụkọ kwa abalị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 21,
      "token_splits": [
        "▁N",
        "ne",
        "▁",
        "nne",
        "▁m",
        "▁na",
        "-",
        "ak",
        "ọ",
        "r",
        "ọ",
        "▁any",
        "ị",
        "▁ak",
        "ụ",
        "k",
        "ọ",
        "▁kwa",
        "▁a",
        "bal",
        "ị"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "My",
        " grand",
        "m",
        "ama",
        " de",
        "y",
        " tell",
        " us",
        " story",
        " every",
        " night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "My",
        " grand",
        "m",
        "ama",
        " dey",
        " tell",
        " us",
        " story",
        " every",
        " night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.429
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "My",
        "Ġgrand",
        "m",
        "ama",
        "Ġde",
        "y",
        "Ġtell",
        "Ġus",
        "Ġstory",
        "Ġevery",
        "Ġnight"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "My",
        "▁grand",
        "mama",
        "▁dey",
        "▁tell",
        "▁us",
        "▁story",
        "▁every",
        "▁night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁My",
        "▁grand",
        "m",
        "ama",
        "▁de",
        "y",
        "▁tell",
        "▁us",
        "▁story",
        "▁every",
        "▁night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "My",
        "grand",
        "##ma",
        "##ma",
        "de",
        "##y",
        "tell",
        "us",
        "story",
        "every",
        "night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 42,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey tell us story every night",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁My",
        "▁grand",
        "mama",
        "▁de",
        "y",
        "▁tell",
        "▁us",
        "▁story",
        "▁every",
        "▁night"
      ],
      "english_equivalent": "My grandmother told us stories every night",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "The",
        " rainy",
        " season",
        " has",
        " started",
        " and",
        " the",
        " farmers",
        " are",
        " happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "The",
        " rainy",
        " season",
        " has",
        " started",
        " and",
        " the",
        " farmers",
        " are",
        " happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "The",
        "Ġrainy",
        "Ġseason",
        "Ġhas",
        "Ġstarted",
        "Ġand",
        "Ġthe",
        "Ġfarmers",
        "Ġare",
        "Ġhappy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "The",
        "▁rainy",
        "▁season",
        "▁has",
        "▁started",
        "▁and",
        "▁the",
        "▁farmers",
        "▁are",
        "▁happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁The",
        "▁rain",
        "y",
        "▁season",
        "▁has",
        "▁started",
        "▁and",
        "▁the",
        "▁farmers",
        "▁are",
        "▁happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "rain",
        "##y",
        "season",
        "has",
        "started",
        "and",
        "the",
        "farmers",
        "are",
        "happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The rainy season has started and the farmers are happy",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁The",
        "▁rain",
        "y",
        "▁season",
        "▁has",
        "▁started",
        "▁and",
        "▁the",
        "▁farmer",
        "s",
        "▁are",
        "▁happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 34,
      "token_splits": [
        "À",
        "k",
        "ó",
        "k",
        "ò",
        " �",
        "�",
        "j",
        "ò",
        " ti",
        " b",
        "�",
        "�",
        "̀",
        "r",
        "�",
        "�",
        "̀",
        ",",
        " à",
        "w",
        "ọn",
        " à",
        "gb",
        "�",
        "�",
        "̀",
        " s",
        "ì",
        " ",
        "ń",
        " y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.4
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 22,
      "token_splits": [
        "À",
        "k",
        "ók",
        "ò",
        " ò",
        "j",
        "ò",
        " ti",
        " bẹ",
        "̀",
        "rẹ",
        "̀",
        ",",
        " àwọn",
        " à",
        "gbẹ",
        "̀",
        " sì",
        " ń",
        " y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.2
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 31,
      "token_splits": [
        "ÃĢ",
        "k",
        "Ã³",
        "k",
        "Ã²",
        "ĠÃ",
        "²",
        "j",
        "Ã²",
        "Ġti",
        "Ġb",
        "áº¹",
        "ÌĢ",
        "r",
        "áº¹",
        "ÌĢ",
        ",",
        "ĠÃł",
        "w",
        "á»įn",
        "ĠÃł",
        "gb",
        "áº¹",
        "ÌĢ",
        "Ġs",
        "Ã¬",
        "Ġ",
        "ÅĦ",
        "Ġy",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.1
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 28,
      "token_splits": [
        "À",
        "k",
        "ók",
        "ò",
        "▁ò",
        "j",
        "ò",
        "▁ti",
        "▁b",
        "ẹ",
        "̀",
        "r",
        "ẹ",
        "̀",
        ",",
        "▁à",
        "w",
        "ọn",
        "▁à",
        "gb",
        "ẹ",
        "̀",
        "▁sì",
        "▁",
        "ń",
        "▁y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.8
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 38,
      "token_splits": [
        "▁À",
        "k",
        "ó",
        "k",
        "ò",
        "▁",
        "ò",
        "j",
        "ò",
        "▁ti",
        "▁b",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        ",",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁à",
        "gb",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "▁s",
        "ì",
        "▁",
        "ń",
        "▁y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.455
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 22,
      "token_splits": [
        "À",
        "##kó",
        "##k",
        "##ò",
        "ò",
        "##j",
        "##ò",
        "ti",
        "b",
        "##ẹ̀",
        "##r",
        "##ẹ̀",
        ",",
        "àwọn",
        "à",
        "##g",
        "##b",
        "##ẹ̀",
        "sì",
        "ń",
        "y",
        "##ọ̀"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àkókò òjò ti bẹ̀rẹ̀, àwọn àgbẹ̀ sì ń yọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 30,
      "token_splits": [
        "▁À",
        "k",
        "ók",
        "ò",
        "▁",
        "ò",
        "j",
        "ò",
        "▁ti",
        "▁b",
        "ẹ",
        "̀",
        "r",
        "ẹ",
        "̀",
        ",",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁à",
        "gb",
        "ẹ",
        "̀",
        "▁sì",
        "▁",
        "ń",
        "▁y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 18,
      "token_splits": [
        "L",
        "ok",
        "acin",
        " dam",
        "ina",
        " ya",
        " far",
        "a",
        " k",
        "uma",
        " man",
        "oma",
        " sun",
        "a",
        " far",
        "in",
        " c",
        "iki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.8
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "Lok",
        "acin",
        " dam",
        "ina",
        " ya",
        " fara",
        " kuma",
        " man",
        "oma",
        " suna",
        " farin",
        " ciki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 18,
      "token_splits": [
        "L",
        "ok",
        "acin",
        "Ġdam",
        "ina",
        "Ġya",
        "Ġfar",
        "a",
        "Ġk",
        "uma",
        "Ġman",
        "oma",
        "Ġsun",
        "a",
        "Ġfar",
        "in",
        "Ġc",
        "iki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.8
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 14,
      "token_splits": [
        "Lok",
        "acin",
        "▁dam",
        "ina",
        "▁ya",
        "▁fara",
        "▁kuma",
        "▁mano",
        "ma",
        "▁suna",
        "▁far",
        "in",
        "▁c",
        "iki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 19,
      "token_splits": [
        "▁L",
        "ok",
        "ac",
        "in",
        "▁dam",
        "ina",
        "▁ya",
        "▁far",
        "a",
        "▁k",
        "uma",
        "▁man",
        "oma",
        "▁sun",
        "a",
        "▁far",
        "in",
        "▁c",
        "iki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.727
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 18,
      "token_splits": [
        "Lok",
        "##aci",
        "##n",
        "dam",
        "##ina",
        "ya",
        "far",
        "##a",
        "ku",
        "##ma",
        "mano",
        "##ma",
        "sun",
        "##a",
        "far",
        "##in",
        "ci",
        "##ki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.636
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Lokacin damina ya fara kuma manoma suna farin ciki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁Loka",
        "cin",
        "▁da",
        "mina",
        "▁ya",
        "▁fara",
        "▁kuma",
        "▁mano",
        "ma",
        "▁suna",
        "▁farin",
        "▁ciki"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 25,
      "token_splits": [
        "O",
        "ge",
        " mm",
        "iri",
        " e",
        "bid",
        "ola",
        " na",
        " nd",
        "ị",
        " ",
        "ọ",
        "r",
        "ụ",
        " ug",
        "bo",
        " n",
        "were",
        " a",
        "�",
        "�",
        "�",
        "ụ",
        "r",
        "ị"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 16,
      "token_splits": [
        "O",
        "ge",
        " mmiri",
        " eb",
        "id",
        "ola",
        " na",
        " ndị",
        " ọrụ",
        " ug",
        "bo",
        " nwere",
        " a",
        "ṅ",
        "ụ",
        "rị"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 24,
      "token_splits": [
        "O",
        "ge",
        "Ġmm",
        "iri",
        "Ġe",
        "bid",
        "ola",
        "Ġna",
        "Ġnd",
        "á»ĭ",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        "Ġug",
        "bo",
        "Ġn",
        "were",
        "Ġa",
        "á¹",
        "ħ",
        "á»¥",
        "r",
        "á»ĭ"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 22,
      "token_splits": [
        "O",
        "ge",
        "▁m",
        "miri",
        "▁eb",
        "ido",
        "la",
        "▁na",
        "▁nd",
        "ị",
        "▁ọ",
        "r",
        "ụ",
        "▁ug",
        "bo",
        "▁n",
        "were",
        "▁a",
        "ṅ",
        "ụ",
        "r",
        "ị"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.2
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 24,
      "token_splits": [
        "▁O",
        "ge",
        "▁mm",
        "iri",
        "▁e",
        "bid",
        "ola",
        "▁na",
        "▁nd",
        "ị",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁u",
        "g",
        "bo",
        "▁n",
        "were",
        "▁a",
        "ṅ",
        "ụ",
        "r",
        "ị"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.182
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 20,
      "token_splits": [
        "Og",
        "##e",
        "mm",
        "##iri",
        "e",
        "##bido",
        "##la",
        "na",
        "n",
        "##d",
        "##ị",
        "ọ",
        "##r",
        "##ụ",
        "ug",
        "##bo",
        "n",
        "##wer",
        "##e",
        "[UNK]"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.818
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Oge mmiri ebidola na ndị ọrụ ugbo nwere aṅụrị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 25,
      "token_splits": [
        "▁Og",
        "e",
        "▁mm",
        "iri",
        "▁e",
        "bido",
        "la",
        "▁na",
        "▁",
        "nd",
        "ị",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁ug",
        "bo",
        "▁n",
        "wer",
        "e",
        "▁a",
        "ṅ",
        "ụ",
        "r",
        "ị"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.083
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Rain",
        " season",
        " don",
        " start",
        " and",
        " de",
        " farmers",
        " dem",
        " de",
        "y",
        " happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Rain",
        " season",
        " don",
        " start",
        " and",
        " de",
        " farmers",
        " dem",
        " dey",
        " happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "Rain",
        "Ġseason",
        "Ġdon",
        "Ġstart",
        "Ġand",
        "Ġde",
        "Ġfarmers",
        "Ġdem",
        "Ġde",
        "y",
        "Ġhappy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "Rain",
        "▁season",
        "▁don",
        "▁start",
        "▁and",
        "▁de",
        "▁farmers",
        "▁dem",
        "▁dey",
        "▁happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Rain",
        "▁season",
        "▁don",
        "▁start",
        "▁and",
        "▁de",
        "▁farmers",
        "▁dem",
        "▁de",
        "y",
        "▁happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Rain",
        "season",
        "don",
        "start",
        "and",
        "de",
        "farmers",
        "dem",
        "de",
        "##y",
        "happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 43,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Rain season don start and de farmers dem dey happy",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁Rain",
        "▁season",
        "▁don",
        "▁start",
        "▁and",
        "▁de",
        "▁farmer",
        "s",
        "▁dem",
        "▁de",
        "y",
        "▁happy"
      ],
      "english_equivalent": "The rainy season has started and the farmers are happy",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "N",
        "igeria",
        " is",
        " the",
        " most",
        " populous",
        " country",
        " in",
        " Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 8,
      "token_splits": [
        "Nigeria",
        " is",
        " the",
        " most",
        " populous",
        " country",
        " in",
        " Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "N",
        "igeria",
        "Ġis",
        "Ġthe",
        "Ġmost",
        "Ġpopulous",
        "Ġcountry",
        "Ġin",
        "ĠAfrica"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "Nigeria",
        "▁is",
        "▁the",
        "▁most",
        "▁populous",
        "▁country",
        "▁in",
        "▁Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁Nigeria",
        "▁is",
        "▁the",
        "▁most",
        "▁pop",
        "ulous",
        "▁country",
        "▁in",
        "▁Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "Nigeria",
        "is",
        "the",
        "most",
        "pop",
        "##ulo",
        "##us",
        "country",
        "in",
        "Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nigeria is the most populous country in Africa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁Nigeria",
        "▁is",
        "▁the",
        "▁most",
        "▁populo",
        "us",
        "▁country",
        "▁in",
        "▁Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 35,
      "token_splits": [
        "N",
        "à",
        "ì",
        "j",
        "í",
        "r",
        "í",
        "à",
        " ni",
        " or",
        "í",
        "l",
        "�",
        "�",
        "̀",
        "-",
        "è",
        "d",
        "è",
        " t",
        "ó",
        " p",
        "ọ",
        "̀",
        " j",
        "ù",
        " l",
        "ọ",
        " n",
        "í",
        " Á",
        "fr",
        "í",
        "k",
        "à"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.889
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 27,
      "token_splits": [
        "N",
        "à",
        "ì",
        "j",
        "ír",
        "í",
        "à",
        " ni",
        " or",
        "íl",
        "ẹ",
        "̀",
        "-",
        "è",
        "d",
        "è",
        " tó",
        " pọ",
        "̀",
        " j",
        "ù",
        " lọ",
        " ní",
        " Á",
        "fr",
        "ík",
        "à"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 3.375
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 30,
      "token_splits": [
        "N",
        "Ãł",
        "Ã¬",
        "jÃŃ",
        "r",
        "ÃŃ",
        "Ãł",
        "Ġni",
        "Ġor",
        "ÃŃl",
        "áº¹",
        "ÌĢ",
        "-",
        "Ã¨",
        "d",
        "Ã¨",
        "Ġt",
        "Ã³",
        "Ġp",
        "á»į",
        "ÌĢ",
        "Ġj",
        "Ã¹",
        "Ġl",
        "á»į",
        "ĠnÃŃ",
        "ĠÃģ",
        "fr",
        "ÃŃk",
        "Ãł"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 27,
      "token_splits": [
        "N",
        "à",
        "ì",
        "jí",
        "rí",
        "à",
        "▁ni",
        "▁or",
        "íl",
        "ẹ",
        "̀",
        "-",
        "è",
        "d",
        "è",
        "▁tó",
        "▁p",
        "ọ",
        "̀",
        "▁j",
        "ù",
        "▁lọ",
        "▁ní",
        "▁Á",
        "fr",
        "ík",
        "à"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 3.375
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 34,
      "token_splits": [
        "▁N",
        "à",
        "ì",
        "j",
        "ír",
        "í",
        "à",
        "▁ni",
        "▁or",
        "íl",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "-",
        "è",
        "d",
        "è",
        "▁t",
        "ó",
        "▁p",
        "ọ",
        "̀",
        "▁j",
        "ù",
        "▁l",
        "ọ",
        "▁n",
        "í",
        "▁Á",
        "fr",
        "í",
        "k",
        "à"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.778
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 26,
      "token_splits": [
        "N",
        "##à",
        "##ì",
        "##jí",
        "##rí",
        "##à",
        "ni",
        "or",
        "##íl",
        "##ẹ̀",
        "-",
        "è",
        "##d",
        "##è",
        "tó",
        "p",
        "##ọ̀",
        "j",
        "##ù",
        "l",
        "##ọ",
        "ní",
        "Á",
        "##fr",
        "##ík",
        "##à"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.6
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nàìjíríà ni orílẹ̀-èdè tó pọ̀ jù lọ ní Áfríkà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 28,
      "token_splits": [
        "▁N",
        "à",
        "ì",
        "jí",
        "rí",
        "à",
        "▁ni",
        "▁or",
        "íl",
        "ẹ",
        "̀",
        "-",
        "è",
        "d",
        "è",
        "▁tó",
        "▁p",
        "ọ",
        "̀",
        "▁j",
        "ù",
        "▁l",
        "ọ",
        "▁ní",
        "▁Á",
        "frí",
        "k",
        "à"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.111
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "N",
        "ij",
        "eri",
        "ya",
        " it",
        "a",
        " ce",
        " ",
        "�",
        "�",
        "asa",
        " m",
        "afi",
        " y",
        "awan",
        " j",
        "ama",
        "'a",
        " a",
        " Af",
        "ir",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.444
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "N",
        "ij",
        "eriya",
        " ita",
        " ce",
        " ƙ",
        "asa",
        " mafi",
        " y",
        "awan",
        " jama",
        "'a",
        " a",
        " Af",
        "irka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.875
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 22,
      "token_splits": [
        "N",
        "ij",
        "eri",
        "ya",
        "Ġit",
        "a",
        "Ġce",
        "Ġ",
        "Æ",
        "Ļ",
        "asa",
        "Ġm",
        "afi",
        "Ġy",
        "awan",
        "Ġj",
        "ama",
        "'a",
        "Ġa",
        "ĠAf",
        "ir",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.444
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 19,
      "token_splits": [
        "N",
        "ij",
        "eri",
        "ya",
        "▁ita",
        "▁ce",
        "▁ƙ",
        "asa",
        "▁ma",
        "fi",
        "▁y",
        "awan",
        "▁jama",
        "'",
        "a",
        "▁a",
        "▁Af",
        "ir",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 2.375
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 24,
      "token_splits": [
        "▁N",
        "ij",
        "eri",
        "ya",
        "▁it",
        "a",
        "▁ce",
        "▁",
        "<0xC6>",
        "<0x99>",
        "asa",
        "▁m",
        "afi",
        "▁y",
        "aw",
        "an",
        "▁j",
        "ama",
        "'",
        "a",
        "▁a",
        "▁Af",
        "ir",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 19,
      "token_splits": [
        "Nije",
        "##riy",
        "##a",
        "it",
        "##a",
        "ce",
        "[UNK]",
        "ma",
        "##fi",
        "ya",
        "##wan",
        "jam",
        "##a",
        "'",
        "a",
        "a",
        "Af",
        "##ir",
        "##ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.9
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nijeriya ita ce ƙasa mafi yawan jama'a a Afirka",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Nije",
        "riya",
        "▁ita",
        "▁ce",
        "▁",
        "ƙ",
        "asa",
        "▁mafi",
        "▁ya",
        "wan",
        "▁ja",
        "ma",
        "'",
        "a",
        "▁a",
        "▁Afirka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.778
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "Na",
        "ị",
        "j",
        "ir",
        "ị",
        "a",
        " b",
        "ụ",
        " ob",
        "odo",
        " k",
        "ach",
        "as",
        "ị",
        " mm",
        "ad",
        "ụ",
        " n",
        "'A",
        "fr",
        "ị",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.444
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "Na",
        "ị",
        "jir",
        "ịa",
        " bụ",
        " obodo",
        " kachasị",
        " mmadụ",
        " n",
        "'A",
        "fr",
        "ị",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.625
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 20,
      "token_splits": [
        "Na",
        "á»ĭ",
        "j",
        "ir",
        "á»ĭa",
        "Ġbá»¥",
        "Ġob",
        "odo",
        "Ġk",
        "ach",
        "as",
        "á»ĭ",
        "Ġmm",
        "ad",
        "á»¥",
        "Ġn",
        "'A",
        "fr",
        "á»ĭ",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.222
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 20,
      "token_splits": [
        "Na",
        "ị",
        "jir",
        "ịa",
        "▁bụ",
        "▁ob",
        "odo",
        "▁k",
        "ach",
        "as",
        "ị",
        "▁m",
        "mad",
        "ụ",
        "▁n",
        "'",
        "A",
        "fr",
        "ị",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 23,
      "token_splits": [
        "▁Na",
        "ị",
        "j",
        "ir",
        "ị",
        "a",
        "▁b",
        "ụ",
        "▁ob",
        "odo",
        "▁k",
        "ach",
        "as",
        "ị",
        "▁mm",
        "ad",
        "ụ",
        "▁n",
        "'",
        "A",
        "fr",
        "ị",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.556
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "Na",
        "##ị",
        "##jir",
        "##ị",
        "##a",
        "b",
        "##ụ",
        "ob",
        "##odo",
        "ka",
        "##chas",
        "##ị",
        "mm",
        "##ad",
        "##ụ",
        "n",
        "'",
        "Af",
        "##r",
        "##ị",
        "##ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.1
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Naịjirịa bụ obodo kachasị mmadụ n'Afrịka",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 21,
      "token_splits": [
        "▁Na",
        "ị",
        "jir",
        "ị",
        "a",
        "▁b",
        "ụ",
        "▁obo",
        "do",
        "▁ka",
        "chas",
        "ị",
        "▁mm",
        "ad",
        "ụ",
        "▁n",
        "'",
        "A",
        "fr",
        "ị",
        "ka"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "N",
        "igeria",
        " na",
        " de",
        " country",
        " we",
        "y",
        " get",
        " de",
        " most",
        " people",
        " for",
        " Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.444
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "Nigeria",
        " na",
        " de",
        " country",
        " we",
        "y",
        " get",
        " de",
        " most",
        " people",
        " for",
        " Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "N",
        "igeria",
        "Ġna",
        "Ġde",
        "Ġcountry",
        "Ġwe",
        "y",
        "Ġget",
        "Ġde",
        "Ġmost",
        "Ġpeople",
        "Ġfor",
        "ĠAfrica"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.444
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "Nigeria",
        "▁na",
        "▁de",
        "▁country",
        "▁wey",
        "▁get",
        "▁de",
        "▁most",
        "▁people",
        "▁for",
        "▁Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.375
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁Nigeria",
        "▁na",
        "▁de",
        "▁country",
        "▁we",
        "y",
        "▁get",
        "▁de",
        "▁most",
        "▁people",
        "▁for",
        "▁Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "Nigeria",
        "na",
        "de",
        "country",
        "we",
        "##y",
        "get",
        "de",
        "most",
        "people",
        "for",
        "Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 44,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nigeria na de country wey get de most people for Africa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁Nigeria",
        "▁na",
        "▁de",
        "▁country",
        "▁we",
        "y",
        "▁get",
        "▁de",
        "▁most",
        "▁people",
        "▁for",
        "▁Africa"
      ],
      "english_equivalent": "Nigeria is the most populous country in Africa",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Education",
        " is",
        " the",
        " key",
        " to",
        " a",
        " better",
        " future",
        " for",
        " our",
        " youth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Education",
        " is",
        " the",
        " key",
        " to",
        " a",
        " better",
        " future",
        " for",
        " our",
        " youth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "Education",
        "Ġis",
        "Ġthe",
        "Ġkey",
        "Ġto",
        "Ġa",
        "Ġbetter",
        "Ġfuture",
        "Ġfor",
        "Ġour",
        "Ġyouth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "Education",
        "▁is",
        "▁the",
        "▁key",
        "▁to",
        "▁a",
        "▁better",
        "▁future",
        "▁for",
        "▁our",
        "▁youth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Education",
        "▁is",
        "▁the",
        "▁key",
        "▁to",
        "▁a",
        "▁better",
        "▁future",
        "▁for",
        "▁our",
        "▁youth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Education",
        "is",
        "the",
        "key",
        "to",
        "a",
        "better",
        "future",
        "for",
        "our",
        "youth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Education is the key to a better future for our youth",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Education",
        "▁is",
        "▁the",
        "▁key",
        "▁to",
        "▁a",
        "▁better",
        "▁future",
        "▁for",
        "▁our",
        "▁youth"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 43,
      "token_splits": [
        "�",
        "�",
        "̀",
        "k",
        "ọ",
        "́",
        " ni",
        " k",
        "ọ",
        "́",
        "k",
        "ọ",
        "́",
        "r",
        "ọ",
        "́",
        " sí",
        " ",
        "ọ",
        "j",
        "ọ",
        "́",
        " ",
        "ọ",
        "̀",
        "la",
        " t",
        "ó",
        " d",
        "á",
        "ra",
        " f",
        "ún",
        " à",
        "w",
        "ọn",
        " ",
        "ọ",
        "̀",
        "d",
        "ọ",
        "́",
        " wa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.909
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 28,
      "token_splits": [
        "Ẹ",
        "̀",
        "kọ",
        "́",
        " ni",
        " kọ",
        "́",
        "kọ",
        "́",
        "rọ",
        "́",
        " sí",
        " ọjọ",
        "́",
        " ọ",
        "̀",
        "la",
        " tó",
        " dá",
        "ra",
        " fún",
        " àwọn",
        " ọ",
        "̀",
        "d",
        "ọ",
        "́",
        " wa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.545
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 42,
      "token_splits": [
        "áº",
        "¸",
        "ÌĢ",
        "k",
        "á»į",
        "Ìģ",
        "Ġni",
        "Ġk",
        "á»į",
        "Ìģ",
        "k",
        "á»į",
        "Ìģ",
        "r",
        "á»į",
        "Ìģ",
        "ĠsÃŃ",
        "Ġ",
        "á»į",
        "j",
        "á»į",
        "Ìģ",
        "Ġ",
        "á»į",
        "ÌĢ",
        "la",
        "Ġt",
        "Ã³",
        "ĠdÃ¡",
        "ra",
        "Ġf",
        "Ãºn",
        "ĠÃł",
        "w",
        "á»įn",
        "Ġ",
        "á»į",
        "ÌĢ",
        "d",
        "á»į",
        "Ìģ",
        "Ġwa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.818
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 37,
      "token_splits": [
        "Ẹ",
        "̀",
        "k",
        "ọ",
        "́",
        "▁ni",
        "▁k",
        "ọ",
        "́",
        "k",
        "ọ",
        "́",
        "r",
        "ọ",
        "́",
        "▁sí",
        "▁ọ",
        "j",
        "ọ",
        "́",
        "▁ọ",
        "̀",
        "la",
        "▁tó",
        "▁dá",
        "ra",
        "▁f",
        "ún",
        "▁à",
        "w",
        "ọn",
        "▁ọ",
        "̀",
        "d",
        "ọ",
        "́",
        "▁wa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.364
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 45,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "̀",
        "k",
        "ọ",
        "́",
        "▁ni",
        "▁k",
        "ọ",
        "́",
        "k",
        "ọ",
        "́",
        "r",
        "ọ",
        "́",
        "▁sí",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁",
        "ọ",
        "̀",
        "la",
        "▁t",
        "ó",
        "▁d",
        "ára",
        "▁f",
        "ún",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁",
        "ọ",
        "̀",
        "d",
        "ọ",
        "́",
        "▁wa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 4.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 29,
      "token_splits": [
        "Ẹ",
        "##̀",
        "##k",
        "##ọ́",
        "ni",
        "k",
        "##ọ́",
        "##k",
        "##ọ́",
        "##r",
        "##ọ́",
        "sí",
        "ọ",
        "##j",
        "##ọ́",
        "ọ",
        "##̀",
        "##la",
        "tó",
        "dá",
        "##ra",
        "f",
        "##ún",
        "àwọn",
        "ọ",
        "##̀",
        "##d",
        "##ọ́",
        "wa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.636
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ̀kọ́ ni kọ́kọ́rọ́ sí ọjọ́ ọ̀la tó dára fún àwọn ọ̀dọ́ wa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 42,
      "token_splits": [
        "▁",
        "Ẹ",
        "̀",
        "k",
        "ọ",
        "́",
        "▁ni",
        "▁k",
        "ọ",
        "́",
        "k",
        "ọ",
        "́",
        "r",
        "ọ",
        "́",
        "▁sí",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁",
        "ọ",
        "̀",
        "la",
        "▁tó",
        "▁d",
        "ára",
        "▁fú",
        "n",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁",
        "ọ",
        "̀",
        "d",
        "ọ",
        "́",
        "▁wa"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.818
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "I",
        "lim",
        "i",
        " sh",
        "i",
        " ne",
        " m",
        "ab",
        "u",
        "�",
        "�",
        "in",
        " ky",
        "ak",
        "ky",
        "awan",
        " mak",
        "om",
        "ar",
        " mat",
        "asan",
        " mu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 18,
      "token_splits": [
        "I",
        "lim",
        "i",
        " shi",
        " ne",
        " m",
        "abu",
        "ɗ",
        "in",
        " ky",
        "ak",
        "ky",
        "awan",
        " mak",
        "omar",
        " mat",
        "asan",
        " mu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.636
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 22,
      "token_splits": [
        "I",
        "lim",
        "i",
        "Ġsh",
        "i",
        "Ġne",
        "Ġm",
        "ab",
        "u",
        "É",
        "Ĺ",
        "in",
        "Ġky",
        "ak",
        "ky",
        "awan",
        "Ġmak",
        "om",
        "ar",
        "Ġmat",
        "asan",
        "Ġmu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "Ili",
        "mi",
        "▁shi",
        "▁ne",
        "▁m",
        "abu",
        "ɗ",
        "in",
        "▁k",
        "yak",
        "ky",
        "awan",
        "▁ma",
        "kom",
        "ar",
        "▁mat",
        "asan",
        "▁mu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.636
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 25,
      "token_splits": [
        "▁Il",
        "imi",
        "▁sh",
        "i",
        "▁ne",
        "▁m",
        "ab",
        "u",
        "<0xC9>",
        "<0x97>",
        "in",
        "▁k",
        "y",
        "ak",
        "ky",
        "aw",
        "an",
        "▁m",
        "ak",
        "om",
        "ar",
        "▁mat",
        "as",
        "an",
        "▁mu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.273
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "Il",
        "##imi",
        "shi",
        "ne",
        "[UNK]",
        "ky",
        "##ak",
        "##ky",
        "##awan",
        "ma",
        "##kom",
        "##ar",
        "mata",
        "##san",
        "mu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.364
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ilimi shi ne mabuɗin kyakkyawan makomar matasan mu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 18,
      "token_splits": [
        "▁I",
        "limi",
        "▁shi",
        "▁ne",
        "▁ma",
        "bu",
        "ɗ",
        "in",
        "▁ky",
        "ak",
        "ky",
        "awan",
        "▁ma",
        "kom",
        "ar",
        "▁mata",
        "san",
        "▁mu"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.636
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 35,
      "token_splits": [
        "Ag",
        "ụ",
        "mak",
        "w",
        "ụ",
        "kw",
        "ọ",
        " b",
        "ụ",
        " isi",
        " i",
        "he",
        " maka",
        " ",
        "ọ",
        "d",
        "ị",
        "n",
        "ihu",
        " ka",
        " m",
        "ma",
        " n",
        "ke",
        " ",
        "ụ",
        "m",
        "ụ",
        " ok",
        "or",
        "ob",
        "ị",
        "a",
        " any",
        "ị"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.182
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 21,
      "token_splits": [
        "Ag",
        "ụ",
        "mak",
        "wụ",
        "kwọ",
        " bụ",
        " isi",
        " ihe",
        " maka",
        " ọdị",
        "ni",
        "hu",
        " ka",
        " mma",
        " nke",
        " ụmụ",
        " ok",
        "or",
        "ob",
        "ịa",
        " anyị"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.909
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 33,
      "token_splits": [
        "Ag",
        "á»¥",
        "mak",
        "w",
        "á»¥",
        "kw",
        "á»į",
        "Ġbá»¥",
        "Ġisi",
        "Ġi",
        "he",
        "Ġmaka",
        "Ġ",
        "á»į",
        "d",
        "á»ĭ",
        "n",
        "ihu",
        "Ġka",
        "Ġm",
        "ma",
        "Ġn",
        "ke",
        "Ġ",
        "á»¥",
        "m",
        "á»¥",
        "Ġok",
        "or",
        "ob",
        "á»ĭa",
        "Ġany",
        "á»ĭ"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 29,
      "token_splits": [
        "Ag",
        "ụ",
        "mak",
        "w",
        "ụ",
        "kw",
        "ọ",
        "▁bụ",
        "▁isi",
        "▁ihe",
        "▁maka",
        "▁ọ",
        "d",
        "ị",
        "ni",
        "hu",
        "▁ka",
        "▁mma",
        "▁nke",
        "▁",
        "ụ",
        "m",
        "ụ",
        "▁ok",
        "or",
        "ob",
        "ịa",
        "▁any",
        "ị"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.636
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 38,
      "token_splits": [
        "▁Ag",
        "ụ",
        "m",
        "ak",
        "w",
        "ụ",
        "kw",
        "ọ",
        "▁b",
        "ụ",
        "▁is",
        "i",
        "▁i",
        "he",
        "▁m",
        "aka",
        "▁",
        "ọ",
        "d",
        "ị",
        "ni",
        "hu",
        "▁ka",
        "▁m",
        "ma",
        "▁n",
        "ke",
        "▁",
        "ụ",
        "m",
        "ụ",
        "▁ok",
        "or",
        "ob",
        "ị",
        "a",
        "▁any",
        "ị"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.455
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 36,
      "token_splits": [
        "A",
        "##g",
        "##ụ",
        "##mak",
        "##w",
        "##ụ",
        "##k",
        "##w",
        "##ọ",
        "b",
        "##ụ",
        "is",
        "##i",
        "ih",
        "##e",
        "maka",
        "ọ",
        "##d",
        "##ị",
        "##nih",
        "##u",
        "ka",
        "mm",
        "##a",
        "n",
        "##ke",
        "ụ",
        "##m",
        "##ụ",
        "oko",
        "##ro",
        "##b",
        "##ị",
        "##a",
        "any",
        "##ị"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.273
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Agụmakwụkwọ bụ isi ihe maka ọdịnihu ka mma nke ụmụ okorobịa anyị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 34,
      "token_splits": [
        "▁Ag",
        "ụ",
        "mak",
        "w",
        "ụ",
        "kw",
        "ọ",
        "▁b",
        "ụ",
        "▁isi",
        "▁i",
        "he",
        "▁maka",
        "▁",
        "ọ",
        "d",
        "ị",
        "nih",
        "u",
        "▁ka",
        "▁",
        "mma",
        "▁",
        "nke",
        "▁",
        "ụ",
        "m",
        "ụ",
        "▁oko",
        "rob",
        "ị",
        "a",
        "▁any",
        "ị"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "Education",
        " na",
        " de",
        " key",
        " to",
        " better",
        " tomorrow",
        " for",
        " our",
        " young",
        " people",
        " dem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "Education",
        " na",
        " de",
        " key",
        " to",
        " better",
        " tomorrow",
        " for",
        " our",
        " young",
        " people",
        " dem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "Education",
        "Ġna",
        "Ġde",
        "Ġkey",
        "Ġto",
        "Ġbetter",
        "Ġtomorrow",
        "Ġfor",
        "Ġour",
        "Ġyoung",
        "Ġpeople",
        "Ġdem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "Education",
        "▁na",
        "▁de",
        "▁key",
        "▁to",
        "▁better",
        "▁tomorrow",
        "▁for",
        "▁our",
        "▁young",
        "▁people",
        "▁dem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁Education",
        "▁na",
        "▁de",
        "▁key",
        "▁to",
        "▁better",
        "▁tomorrow",
        "▁for",
        "▁our",
        "▁young",
        "▁people",
        "▁dem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.091
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "Education",
        "na",
        "de",
        "key",
        "to",
        "better",
        "tom",
        "##or",
        "##row",
        "for",
        "our",
        "young",
        "people",
        "dem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.273
    },
    {
      "entry_id": 45,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Education na de key to better tomorrow for our young people dem",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁Education",
        "▁na",
        "▁de",
        "▁key",
        "▁to",
        "▁better",
        "▁tomorrow",
        "▁for",
        "▁our",
        "▁young",
        "▁people",
        "▁dem"
      ],
      "english_equivalent": "Education is the key to a better future for our youth",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.091
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "The",
        " new",
        " technology",
        " is",
        " transforming",
        " how",
        " businesses",
        " operate",
        " in",
        " Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "The",
        " new",
        " technology",
        " is",
        " transforming",
        " how",
        " businesses",
        " operate",
        " in",
        " Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "The",
        "Ġnew",
        "Ġtechnology",
        "Ġis",
        "Ġtransforming",
        "Ġhow",
        "Ġbusinesses",
        "Ġoperate",
        "Ġin",
        "ĠLagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "The",
        "▁new",
        "▁technology",
        "▁is",
        "▁transforming",
        "▁how",
        "▁businesses",
        "▁operate",
        "▁in",
        "▁Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁The",
        "▁new",
        "▁technology",
        "▁is",
        "▁transform",
        "ing",
        "▁how",
        "▁businesses",
        "▁operate",
        "▁in",
        "▁Lag",
        "os"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "new",
        "technology",
        "is",
        "transform",
        "##ing",
        "how",
        "businesses",
        "operate",
        "in",
        "Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The new technology is transforming how businesses operate in Lagos",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁The",
        "▁new",
        "▁technology",
        "▁is",
        "▁transform",
        "ing",
        "▁how",
        "▁businesses",
        "▁operate",
        "▁in",
        "▁Lago",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 54,
      "token_splits": [
        "�",
        "�",
        "m",
        "ọ",
        "̀",
        " ",
        "�",
        "�",
        "̀",
        "r",
        "ọ",
        " t",
        "unt",
        "un",
        " ",
        "ń",
        " y",
        "í",
        " b",
        "í",
        " à",
        "w",
        "ọn",
        " i",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́",
        " �",
        "�",
        "�",
        "e",
        " ",
        "ń",
        " �",
        "�",
        "�",
        "i",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́",
        " n",
        "í",
        " �",
        "�",
        "k",
        "ó",
        " pad",
        "à"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 5.4
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 24,
      "token_splits": [
        "Ì",
        "mọ",
        "̀",
        " ẹ",
        "̀",
        "rọ",
        " tuntun",
        " ń",
        " y",
        "í",
        " bí",
        " àwọn",
        " iṣẹ",
        "́",
        " ṣe",
        " ń",
        " ṣiṣẹ",
        "́",
        " ní",
        " È",
        "k",
        "ó",
        " pad",
        "à"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 45,
      "token_splits": [
        "ÃĮ",
        "m",
        "á»į",
        "ÌĢ",
        "Ġ",
        "áº¹",
        "ÌĢ",
        "r",
        "á»į",
        "Ġt",
        "unt",
        "un",
        "Ġ",
        "ÅĦ",
        "Ġy",
        "ÃŃ",
        "ĠbÃŃ",
        "ĠÃł",
        "w",
        "á»įn",
        "Ġi",
        "á¹",
        "£",
        "áº¹",
        "Ìģ",
        "Ġá",
        "¹",
        "£",
        "e",
        "Ġ",
        "ÅĦ",
        "Ġá",
        "¹",
        "£i",
        "á¹",
        "£",
        "áº¹",
        "Ìģ",
        "ĠnÃŃ",
        "ĠÃ",
        "Ī",
        "k",
        "Ã³",
        "Ġpad",
        "Ãł"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 39,
      "token_splits": [
        "Ì",
        "m",
        "ọ",
        "̀",
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁tunt",
        "un",
        "▁",
        "ń",
        "▁y",
        "í",
        "▁bí",
        "▁à",
        "w",
        "ọn",
        "▁i",
        "ṣ",
        "ẹ",
        "́",
        "▁",
        "ṣ",
        "e",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "ẹ",
        "́",
        "▁ní",
        "▁È",
        "kó",
        "▁pad",
        "à"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.9
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 51,
      "token_splits": [
        "▁",
        "Ì",
        "m",
        "ọ",
        "̀",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ọ",
        "▁t",
        "unt",
        "un",
        "▁",
        "ń",
        "▁y",
        "í",
        "▁b",
        "í",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁i",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁",
        "ṣ",
        "e",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁n",
        "í",
        "▁È",
        "k",
        "ó",
        "▁pad",
        "à"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 4.25
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 29,
      "token_splits": [
        "Ì",
        "##m",
        "##ọ̀",
        "ẹ",
        "##̀",
        "##r",
        "##ọ",
        "tun",
        "##tun",
        "ń",
        "y",
        "##í",
        "bí",
        "àwọn",
        "i",
        "##ṣ",
        "##ẹ́",
        "ṣ",
        "##e",
        "ń",
        "ṣ",
        "##i",
        "##ṣ",
        "##ẹ́",
        "ní",
        "È",
        "##kó",
        "pad",
        "##à"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.636
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìmọ̀ ẹ̀rọ tuntun ń yí bí àwọn iṣẹ́ ṣe ń ṣiṣẹ́ ní Èkó padà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 41,
      "token_splits": [
        "▁",
        "Ì",
        "m",
        "ọ",
        "̀",
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁tuntu",
        "n",
        "▁",
        "ń",
        "▁y",
        "í",
        "▁bí",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁i",
        "ṣ",
        "ẹ",
        "́",
        "▁",
        "ṣ",
        "e",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "ẹ",
        "́",
        "▁ní",
        "▁È",
        "kó",
        "▁pad",
        "à"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.417
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 23,
      "token_splits": [
        "Sab",
        "u",
        "war",
        " fas",
        "aha",
        " t",
        "ana",
        " can",
        "za",
        " y",
        "add",
        "a",
        " kas",
        "uw",
        "anc",
        "i",
        " ke",
        " g",
        "ud",
        "ana",
        " a",
        " Leg",
        "as"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.3
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "Sab",
        "uwar",
        " fas",
        "aha",
        " tana",
        " can",
        "za",
        " yadda",
        " kas",
        "uw",
        "anci",
        " ke",
        " gud",
        "ana",
        " a",
        " Leg",
        "as"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.7
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 23,
      "token_splits": [
        "Sab",
        "u",
        "war",
        "Ġfas",
        "aha",
        "Ġt",
        "ana",
        "Ġcan",
        "za",
        "Ġy",
        "add",
        "a",
        "Ġkas",
        "uw",
        "anc",
        "i",
        "Ġke",
        "Ġg",
        "ud",
        "ana",
        "Ġa",
        "ĠLeg",
        "as"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.3
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 21,
      "token_splits": [
        "Sab",
        "u",
        "war",
        "▁f",
        "asa",
        "ha",
        "▁tana",
        "▁can",
        "za",
        "▁y",
        "adda",
        "▁kas",
        "u",
        "wan",
        "ci",
        "▁ke",
        "▁gud",
        "ana",
        "▁a",
        "▁Le",
        "gas"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.1
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 25,
      "token_splits": [
        "▁Sab",
        "u",
        "war",
        "▁f",
        "as",
        "aha",
        "▁t",
        "ana",
        "▁can",
        "za",
        "▁y",
        "add",
        "a",
        "▁k",
        "as",
        "uw",
        "anc",
        "i",
        "▁ke",
        "▁g",
        "ud",
        "ana",
        "▁a",
        "▁Leg",
        "as"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.083
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "Sa",
        "##bu",
        "##war",
        "fa",
        "##sah",
        "##a",
        "tan",
        "##a",
        "can",
        "##za",
        "ya",
        "##dda",
        "kas",
        "##uwa",
        "##nci",
        "ke",
        "gud",
        "##ana",
        "a",
        "Lega",
        "##s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.909
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sabuwar fasaha tana canza yadda kasuwanci ke gudana a Legas",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁S",
        "abuwar",
        "▁fasa",
        "ha",
        "▁tana",
        "▁can",
        "za",
        "▁yadda",
        "▁kasuwa",
        "nci",
        "▁ke",
        "▁guda",
        "na",
        "▁a",
        "▁Lega",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 34,
      "token_splits": [
        "Tek",
        "n",
        "ụ",
        "z",
        "ụ",
        " ",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        " na",
        "-ag",
        "ban",
        "we",
        " ot",
        "u",
        " az",
        "ụ",
        "ma",
        "ah",
        "ị",
        "a",
        " si",
        " ar",
        "ụ",
        " ",
        "ọ",
        "r",
        "ụ",
        " na",
        " Leg",
        "ọ",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.4
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 19,
      "token_splits": [
        "Tek",
        "n",
        "ụ",
        "zụ",
        " ọhụrụ",
        " na",
        "-ag",
        "banwe",
        " otu",
        " azụ",
        "ma",
        "ahịa",
        " si",
        " arụ",
        " ọrụ",
        " na",
        " Leg",
        "ọ",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.9
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 33,
      "token_splits": [
        "Tek",
        "n",
        "á»¥",
        "z",
        "á»¥",
        "Ġ",
        "á»į",
        "h",
        "á»¥",
        "r",
        "á»¥",
        "Ġna",
        "-ag",
        "ban",
        "we",
        "Ġot",
        "u",
        "Ġaz",
        "á»¥",
        "ma",
        "ah",
        "á»ĭa",
        "Ġsi",
        "Ġar",
        "á»¥",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        "Ġna",
        "ĠLeg",
        "á»į",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.3
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 31,
      "token_splits": [
        "Tek",
        "n",
        "ụ",
        "z",
        "ụ",
        "▁ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁na",
        "-",
        "ag",
        "ban",
        "we",
        "▁otu",
        "▁az",
        "ụ",
        "ma",
        "ah",
        "ịa",
        "▁si",
        "▁ar",
        "ụ",
        "▁ọ",
        "r",
        "ụ",
        "▁na",
        "▁Leg",
        "ọ",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.1
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 36,
      "token_splits": [
        "▁T",
        "ek",
        "n",
        "ụ",
        "z",
        "ụ",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁na",
        "-",
        "ag",
        "ban",
        "we",
        "▁ot",
        "u",
        "▁az",
        "ụ",
        "ma",
        "ah",
        "ị",
        "a",
        "▁si",
        "▁ar",
        "ụ",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁na",
        "▁Leg",
        "ọ",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 33,
      "token_splits": [
        "Tek",
        "##n",
        "##ụ",
        "##z",
        "##ụ",
        "ọ",
        "##h",
        "##ụ",
        "##r",
        "##ụ",
        "na",
        "-",
        "ag",
        "##ban",
        "##we",
        "ot",
        "##u",
        "az",
        "##ụ",
        "##maa",
        "##hị",
        "##a",
        "si",
        "ar",
        "##ụ",
        "ọ",
        "##r",
        "##ụ",
        "na",
        "Le",
        "##g",
        "##ọ",
        "##s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Teknụzụ ọhụrụ na-agbanwe otu azụmaahịa si arụ ọrụ na Legọs",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 35,
      "token_splits": [
        "▁Tek",
        "n",
        "ụ",
        "z",
        "ụ",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁na",
        "-",
        "ag",
        "ban",
        "we",
        "▁o",
        "tu",
        "▁az",
        "ụ",
        "ma",
        "ah",
        "ị",
        "a",
        "▁si",
        "▁ar",
        "ụ",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁na",
        "▁Leg",
        "ọ",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.917
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "Dis",
        " new",
        " technology",
        " de",
        "y",
        " change",
        " how",
        " business",
        " de",
        "y",
        " run",
        " for",
        " Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.3
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Dis",
        " new",
        " technology",
        " dey",
        " change",
        " how",
        " business",
        " dey",
        " run",
        " for",
        " Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "Dis",
        "Ġnew",
        "Ġtechnology",
        "Ġde",
        "y",
        "Ġchange",
        "Ġhow",
        "Ġbusiness",
        "Ġde",
        "y",
        "Ġrun",
        "Ġfor",
        "ĠLagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.3
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "Dis",
        "▁new",
        "▁technology",
        "▁dey",
        "▁change",
        "▁how",
        "▁business",
        "▁dey",
        "▁run",
        "▁for",
        "▁Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁Dis",
        "▁new",
        "▁technology",
        "▁de",
        "y",
        "▁change",
        "▁how",
        "▁business",
        "▁de",
        "y",
        "▁run",
        "▁for",
        "▁Lag",
        "os"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "Dis",
        "new",
        "technology",
        "de",
        "##y",
        "change",
        "how",
        "business",
        "de",
        "##y",
        "run",
        "for",
        "Lagos"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.182
    },
    {
      "entry_id": 46,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dis new technology dey change how business dey run for Lagos",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁Dis",
        "▁new",
        "▁technology",
        "▁de",
        "y",
        "▁change",
        "▁how",
        "▁business",
        "▁de",
        "y",
        "▁run",
        "▁for",
        "▁Lago",
        "s"
      ],
      "english_equivalent": "The new technology is transforming how businesses operate in Lagos",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "President",
        " Tin",
        "ub",
        "u",
        " addressed",
        " the",
        " nation",
        " on",
        " Independence",
        " Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "President",
        " Tin",
        "ubu",
        " addressed",
        " the",
        " nation",
        " on",
        " Independence",
        " Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "President",
        "ĠTin",
        "ubu",
        "Ġaddressed",
        "Ġthe",
        "Ġnation",
        "Ġon",
        "ĠIndependence",
        "ĠDay"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "President",
        "▁Tinubu",
        "▁addressed",
        "▁the",
        "▁nation",
        "▁on",
        "▁Independence",
        "▁Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁President",
        "▁T",
        "in",
        "ub",
        "u",
        "▁addressed",
        "▁the",
        "▁nation",
        "▁on",
        "▁Independ",
        "ence",
        "▁Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "President",
        "Tin",
        "##ub",
        "##u",
        "addressed",
        "the",
        "nation",
        "on",
        "Independence",
        "Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "President Tinubu addressed the nation on Independence Day",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁President",
        "▁Tin",
        "u",
        "bu",
        "▁address",
        "ed",
        "▁the",
        "▁nation",
        "▁on",
        "▁Independen",
        "ce",
        "▁Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 39,
      "token_splits": [
        "À",
        "à",
        "r",
        "�",
        "�",
        " Tin",
        "ub",
        "u",
        " b",
        "á",
        " or",
        "í",
        "l",
        "�",
        "�",
        "̀",
        "-",
        "è",
        "d",
        "è",
        " s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀",
        " n",
        "í",
        " ",
        "ọ",
        "j",
        "ọ",
        "́",
        " �",
        "�",
        "m",
        "ì",
        "n",
        "ira"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.9
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 25,
      "token_splits": [
        "À",
        "à",
        "rẹ",
        " Tin",
        "ubu",
        " bá",
        " or",
        "íl",
        "ẹ",
        "̀",
        "-",
        "è",
        "d",
        "è",
        " sọ",
        "̀",
        "rọ",
        "̀",
        " ní",
        " ọjọ",
        "́",
        " ò",
        "m",
        "ìn",
        "ira"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.778
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 33,
      "token_splits": [
        "ÃĢ",
        "Ãł",
        "r",
        "áº¹",
        "ĠTin",
        "ubu",
        "Ġb",
        "Ã¡",
        "Ġor",
        "ÃŃl",
        "áº¹",
        "ÌĢ",
        "-",
        "Ã¨",
        "d",
        "Ã¨",
        "Ġs",
        "á»į",
        "ÌĢ",
        "r",
        "á»į",
        "ÌĢ",
        "ĠnÃŃ",
        "Ġ",
        "á»į",
        "j",
        "á»į",
        "Ìģ",
        "ĠÃ",
        "²",
        "m",
        "Ã¬n",
        "ira"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.667
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 30,
      "token_splits": [
        "À",
        "à",
        "r",
        "ẹ",
        "▁Tinubu",
        "▁bá",
        "▁or",
        "íl",
        "ẹ",
        "̀",
        "-",
        "è",
        "d",
        "è",
        "▁s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀",
        "▁ní",
        "▁ọ",
        "j",
        "ọ",
        "́",
        "▁ò",
        "m",
        "ì",
        "ni",
        "ra"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 8,
      "token_ratio_vs_english": 3.75
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 41,
      "token_splits": [
        "▁À",
        "à",
        "r",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "▁T",
        "in",
        "ub",
        "u",
        "▁b",
        "á",
        "▁or",
        "íl",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "-",
        "è",
        "d",
        "è",
        "▁s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀",
        "▁n",
        "í",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁",
        "ò",
        "m",
        "ì",
        "n",
        "ira"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.417
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 28,
      "token_splits": [
        "À",
        "##à",
        "##r",
        "##ẹ",
        "Tin",
        "##ub",
        "##u",
        "bá",
        "or",
        "##íl",
        "##ẹ̀",
        "-",
        "è",
        "##d",
        "##è",
        "s",
        "##ọ̀",
        "##r",
        "##ọ̀",
        "ní",
        "ọ",
        "##j",
        "##ọ́",
        "ò",
        "##m",
        "##ì",
        "##nir",
        "##a"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.8
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ààrẹ Tinubu bá orílẹ̀-èdè sọ̀rọ̀ ní ọjọ́ òmìnira",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 34,
      "token_splits": [
        "▁À",
        "à",
        "r",
        "ẹ",
        "▁Tin",
        "u",
        "bu",
        "▁bá",
        "▁or",
        "íl",
        "ẹ",
        "̀",
        "-",
        "è",
        "d",
        "è",
        "▁s",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀",
        "▁ní",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁",
        "ò",
        "m",
        "ì",
        "ni",
        "ra"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.833
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 24,
      "token_splits": [
        "Sh",
        "ug",
        "aba",
        " Tin",
        "ub",
        "u",
        " ya",
        " yi",
        " jaw",
        "abi",
        " ga",
        " al",
        "'",
        "um",
        "ma",
        " a",
        " ran",
        "ar",
        " '",
        "y",
        "anc",
        "in",
        " k",
        "ai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 19,
      "token_splits": [
        "Sh",
        "ug",
        "aba",
        " Tin",
        "ubu",
        " ya",
        " yi",
        " jaw",
        "abi",
        " ga",
        " al",
        "'um",
        "ma",
        " a",
        " ranar",
        " '",
        "y",
        "ancin",
        " kai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.111
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 23,
      "token_splits": [
        "Sh",
        "ug",
        "aba",
        "ĠTin",
        "ubu",
        "Ġya",
        "Ġyi",
        "Ġjaw",
        "abi",
        "Ġga",
        "Ġal",
        "'",
        "um",
        "ma",
        "Ġa",
        "Ġran",
        "ar",
        "Ġ'",
        "y",
        "anc",
        "in",
        "Ġk",
        "ai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.556
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 19,
      "token_splits": [
        "Sh",
        "ug",
        "aba",
        "▁Tinubu",
        "▁ya",
        "▁yi",
        "▁jaw",
        "abi",
        "▁ga",
        "▁al",
        "'",
        "umma",
        "▁a",
        "▁ran",
        "ar",
        "▁'",
        "yan",
        "cin",
        "▁kai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 8,
      "token_ratio_vs_english": 2.375
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁Sh",
        "ug",
        "aba",
        "▁T",
        "in",
        "ub",
        "u",
        "▁ya",
        "▁y",
        "i",
        "▁jaw",
        "abi",
        "▁ga",
        "▁al",
        "'",
        "um",
        "ma",
        "▁a",
        "▁ran",
        "ar",
        "▁'",
        "y",
        "anc",
        "in",
        "▁k",
        "ai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 23,
      "token_splits": [
        "Shu",
        "##gab",
        "##a",
        "Tin",
        "##ub",
        "##u",
        "ya",
        "yi",
        "jawab",
        "##i",
        "ga",
        "al",
        "'",
        "um",
        "##ma",
        "a",
        "ran",
        "##ar",
        "'",
        "ya",
        "##nci",
        "##n",
        "kai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.3
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Shugaba Tinubu ya yi jawabi ga al'umma a ranar 'yancin kai",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 19,
      "token_splits": [
        "▁Shugaba",
        "▁Tin",
        "u",
        "bu",
        "▁ya",
        "▁yi",
        "▁jawab",
        "i",
        "▁ga",
        "▁al",
        "'",
        "um",
        "ma",
        "▁a",
        "▁ranar",
        "▁'",
        "yan",
        "cin",
        "▁kai"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.583
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 27,
      "token_splits": [
        "On",
        "ye",
        " isi",
        " al",
        "a",
        " Tin",
        "ub",
        "u",
        " kw",
        "uru",
        " ok",
        "wu",
        " nye",
        " m",
        "ba",
        " a",
        " n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        " nn",
        "were",
        " on",
        "we"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.7
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "On",
        "ye",
        " isi",
        " ala",
        " Tin",
        "ubu",
        " kwuru",
        " okwu",
        " nye",
        " mba",
        " a",
        " n",
        "'ụ",
        "bọchị",
        " nn",
        "were",
        " onwe"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.889
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 26,
      "token_splits": [
        "On",
        "ye",
        "Ġisi",
        "Ġal",
        "a",
        "ĠTin",
        "ubu",
        "Ġkw",
        "uru",
        "Ġok",
        "wu",
        "Ġnye",
        "Ġm",
        "ba",
        "Ġa",
        "Ġn",
        "'",
        "á»¥",
        "b",
        "á»į",
        "ch",
        "á»ĭ",
        "Ġnn",
        "were",
        "Ġon",
        "we"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.889
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 23,
      "token_splits": [
        "O",
        "nye",
        "▁isi",
        "▁ala",
        "▁Tinubu",
        "▁kw",
        "uru",
        "▁ok",
        "wu",
        "▁nye",
        "▁mba",
        "▁a",
        "▁n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁nn",
        "were",
        "▁on",
        "we"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 8,
      "token_ratio_vs_english": 2.875
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 31,
      "token_splits": [
        "▁On",
        "ye",
        "▁is",
        "i",
        "▁al",
        "a",
        "▁T",
        "in",
        "ub",
        "u",
        "▁kw",
        "uru",
        "▁ok",
        "w",
        "u",
        "▁n",
        "ye",
        "▁m",
        "ba",
        "▁a",
        "▁n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁nn",
        "were",
        "▁on",
        "we"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.583
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 28,
      "token_splits": [
        "On",
        "##ye",
        "is",
        "##i",
        "ala",
        "Tin",
        "##ub",
        "##u",
        "k",
        "##wu",
        "##ru",
        "ok",
        "##wu",
        "nye",
        "m",
        "##ba",
        "a",
        "n",
        "'",
        "ụ",
        "##b",
        "##ọc",
        "##hị",
        "nn",
        "##wer",
        "##e",
        "on",
        "##we"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.8
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye isi ala Tinubu kwuru okwu nye mba a n'ụbọchị nnwere onwe",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 27,
      "token_splits": [
        "▁O",
        "nye",
        "▁isi",
        "▁ala",
        "▁Tin",
        "u",
        "bu",
        "▁kw",
        "uru",
        "▁ok",
        "wu",
        "▁nye",
        "▁mba",
        "▁a",
        "▁n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁n",
        "n",
        "wer",
        "e",
        "▁on",
        "we"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "President",
        " Tin",
        "ub",
        "u",
        " talk",
        " to",
        " de",
        " country",
        " on",
        " Independence",
        " Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "President",
        " Tin",
        "ubu",
        " talk",
        " to",
        " de",
        " country",
        " on",
        " Independence",
        " Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "President",
        "ĠTin",
        "ubu",
        "Ġtalk",
        "Ġto",
        "Ġde",
        "Ġcountry",
        "Ġon",
        "ĠIndependence",
        "ĠDay"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "President",
        "▁Tinubu",
        "▁talk",
        "▁to",
        "▁de",
        "▁country",
        "▁on",
        "▁Independence",
        "▁Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.125
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁President",
        "▁T",
        "in",
        "ub",
        "u",
        "▁talk",
        "▁to",
        "▁de",
        "▁country",
        "▁on",
        "▁Independ",
        "ence",
        "▁Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "President",
        "Tin",
        "##ub",
        "##u",
        "talk",
        "to",
        "de",
        "country",
        "on",
        "Independence",
        "Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 47,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "President Tinubu talk to de country on Independence Day",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁President",
        "▁Tin",
        "u",
        "bu",
        "▁talk",
        "▁to",
        "▁de",
        "▁country",
        "▁on",
        "▁Independen",
        "ce",
        "▁Day"
      ],
      "english_equivalent": "President Tinubu addressed the nation on Independence Day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "The",
        " j",
        "ollo",
        "f",
        " rice",
        " competition",
        " between",
        " Nigeria",
        " and",
        " Ghana",
        " is",
        " legendary"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "The",
        " j",
        "ollo",
        "f",
        " rice",
        " competition",
        " between",
        " Nigeria",
        " and",
        " Ghana",
        " is",
        " legendary"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "Ġj",
        "ollo",
        "f",
        "Ġrice",
        "Ġcompetition",
        "Ġbetween",
        "ĠNigeria",
        "Ġand",
        "ĠGhana",
        "Ġis",
        "Ġlegendary"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "▁jo",
        "ll",
        "of",
        "▁rice",
        "▁competition",
        "▁between",
        "▁Nigeria",
        "▁and",
        "▁Ghana",
        "▁is",
        "▁legendary"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁The",
        "▁j",
        "ol",
        "lo",
        "f",
        "▁rice",
        "▁competition",
        "▁between",
        "▁Nigeria",
        "▁and",
        "▁Gh",
        "ana",
        "▁is",
        "▁legendary"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "jo",
        "##llo",
        "##f",
        "rice",
        "competition",
        "between",
        "Nigeria",
        "and",
        "Ghana",
        "is",
        "legendary"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The jollof rice competition between Nigeria and Ghana is legendary",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁The",
        "▁jo",
        "llo",
        "f",
        "▁rice",
        "▁competition",
        "▁between",
        "▁Nigeria",
        "▁and",
        "▁Ghana",
        "▁is",
        "▁legenda",
        "ry"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 35,
      "token_splits": [
        "�",
        "�",
        "d",
        "í",
        "je",
        " ir",
        "�",
        "�",
        "s",
        "ì",
        " j",
        "ól",
        "ó",
        "f",
        " lá",
        "à",
        "ár",
        "ín",
        " N",
        "à",
        "ì",
        "j",
        "í",
        "r",
        "í",
        "à",
        " à",
        "ti",
        " Ghana",
        " ti",
        " di",
        " à",
        "k",
        "ày",
        "é"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.917
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 29,
      "token_splits": [
        "Ì",
        "d",
        "í",
        "je",
        " ir",
        "ẹ",
        "s",
        "ì",
        " jól",
        "óf",
        " lá",
        "à",
        "ár",
        "ín",
        " N",
        "à",
        "ì",
        "j",
        "ír",
        "í",
        "à",
        " àti",
        " Ghana",
        " ti",
        " di",
        " à",
        "k",
        "ày",
        "é"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.417
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 31,
      "token_splits": [
        "ÃĮ",
        "dÃŃ",
        "je",
        "Ġir",
        "áº¹",
        "s",
        "Ã¬",
        "Ġj",
        "Ã³l",
        "Ã³",
        "f",
        "ĠlÃ¡",
        "Ãł",
        "Ã¡r",
        "ÃŃn",
        "ĠN",
        "Ãł",
        "Ã¬",
        "jÃŃ",
        "r",
        "ÃŃ",
        "Ãł",
        "ĠÃł",
        "ti",
        "ĠGhana",
        "Ġti",
        "Ġdi",
        "ĠÃł",
        "k",
        "Ãły",
        "Ã©"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.583
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 28,
      "token_splits": [
        "Ì",
        "dí",
        "je",
        "▁ir",
        "ẹ",
        "s",
        "ì",
        "▁jól",
        "óf",
        "▁lá",
        "à",
        "ár",
        "ín",
        "▁N",
        "à",
        "ì",
        "jí",
        "rí",
        "à",
        "▁à",
        "ti",
        "▁Ghana",
        "▁ti",
        "▁di",
        "▁à",
        "k",
        "ày",
        "é"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 38,
      "token_splits": [
        "▁",
        "Ì",
        "d",
        "í",
        "je",
        "▁ir",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "s",
        "ì",
        "▁j",
        "ól",
        "ó",
        "f",
        "▁l",
        "á",
        "à",
        "ár",
        "ín",
        "▁N",
        "à",
        "ì",
        "j",
        "ír",
        "í",
        "à",
        "▁à",
        "ti",
        "▁Gh",
        "ana",
        "▁ti",
        "▁di",
        "▁à",
        "k",
        "à",
        "y",
        "é"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.714
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 29,
      "token_splits": [
        "Ì",
        "##díj",
        "##e",
        "ir",
        "##ẹ",
        "##s",
        "##ì",
        "jól",
        "##ó",
        "##f",
        "lá",
        "##à",
        "##ár",
        "##ín",
        "N",
        "##à",
        "##ì",
        "##jí",
        "##rí",
        "##à",
        "àti",
        "Ghana",
        "ti",
        "di",
        "à",
        "##k",
        "##à",
        "##y",
        "##é"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.417
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìdíje irẹsì jólóf láàárín Nàìjíríà àti Ghana ti di àkàyé",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 31,
      "token_splits": [
        "▁",
        "Ì",
        "díj",
        "e",
        "▁ir",
        "ẹ",
        "s",
        "ì",
        "▁jól",
        "ó",
        "f",
        "▁lá",
        "à",
        "ár",
        "ín",
        "▁N",
        "à",
        "ì",
        "jí",
        "rí",
        "à",
        "▁à",
        "ti",
        "▁Ghana",
        "▁ti",
        "▁di",
        "▁à",
        "k",
        "à",
        "y",
        "é"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.385
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "G",
        "asa",
        " j",
        "ollo",
        "f",
        " rice",
        " ts",
        "akan",
        "in",
        " N",
        "ij",
        "eri",
        "ya",
        " da",
        " Ghana",
        " lab",
        "ari",
        " ne",
        " na",
        " t",
        "ari",
        "hi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.833
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 16,
      "token_splits": [
        "G",
        "asa",
        " j",
        "ollo",
        "f",
        " rice",
        " tsakanin",
        " Nij",
        "eriya",
        " da",
        " Ghana",
        " lab",
        "ari",
        " ne",
        " na",
        " tarihi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 20,
      "token_splits": [
        "G",
        "asa",
        "Ġj",
        "ollo",
        "f",
        "Ġrice",
        "Ġts",
        "akan",
        "in",
        "ĠN",
        "ij",
        "eri",
        "ya",
        "Ġda",
        "ĠGhana",
        "Ġlab",
        "ari",
        "Ġne",
        "Ġna",
        "Ġtarihi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 19,
      "token_splits": [
        "G",
        "asa",
        "▁jo",
        "ll",
        "of",
        "▁rice",
        "▁ts",
        "akan",
        "in",
        "▁Nij",
        "eri",
        "ya",
        "▁da",
        "▁Ghana",
        "▁lab",
        "ari",
        "▁ne",
        "▁na",
        "▁tarihi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.583
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 25,
      "token_splits": [
        "▁G",
        "asa",
        "▁j",
        "ol",
        "lo",
        "f",
        "▁rice",
        "▁ts",
        "ak",
        "an",
        "in",
        "▁N",
        "ij",
        "eri",
        "ya",
        "▁da",
        "▁Gh",
        "ana",
        "▁lab",
        "ari",
        "▁ne",
        "▁na",
        "▁t",
        "ari",
        "hi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.786
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 19,
      "token_splits": [
        "Gas",
        "##a",
        "jo",
        "##llo",
        "##f",
        "rice",
        "t",
        "##saka",
        "##nin",
        "Nije",
        "##riy",
        "##a",
        "da",
        "Ghana",
        "laba",
        "##ri",
        "ne",
        "na",
        "tarihi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.583
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Gasa jollof rice tsakanin Nijeriya da Ghana labari ne na tarihi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Gas",
        "a",
        "▁jo",
        "llo",
        "f",
        "▁rice",
        "▁tsakanin",
        "▁Nije",
        "riya",
        "▁da",
        "▁Ghana",
        "▁la",
        "bari",
        "▁ne",
        "▁na",
        "▁tarihi"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.231
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 33,
      "token_splits": [
        "As",
        "ọ",
        "mp",
        "ị",
        " os",
        "ik",
        "apa",
        " j",
        "ọ",
        "l",
        "ọ",
        "f",
        " n",
        "'",
        "et",
        "iti",
        " Na",
        "ị",
        "j",
        "ir",
        "ị",
        "a",
        " na",
        " Ghana",
        " ab",
        "ụ",
        "r",
        "ụ",
        "la",
        " ak",
        "ụ",
        "k",
        "ọ"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 23,
      "token_splits": [
        "As",
        "ọ",
        "mp",
        "ị",
        " os",
        "ik",
        "apa",
        " j",
        "ọ",
        "lọ",
        "f",
        " n",
        "'etiti",
        " Na",
        "ị",
        "jir",
        "ịa",
        " na",
        " Ghana",
        " ab",
        "ụrụ",
        "la",
        " akụkọ"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.917
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 32,
      "token_splits": [
        "As",
        "á»į",
        "mp",
        "á»ĭ",
        "Ġos",
        "ik",
        "apa",
        "Ġj",
        "á»į",
        "l",
        "á»į",
        "f",
        "Ġn",
        "'",
        "et",
        "iti",
        "ĠNa",
        "á»ĭ",
        "j",
        "ir",
        "á»ĭa",
        "Ġna",
        "ĠGhana",
        "Ġab",
        "á»¥",
        "r",
        "á»¥",
        "la",
        "Ġak",
        "á»¥",
        "k",
        "á»į"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 31,
      "token_splits": [
        "As",
        "ọ",
        "mp",
        "ị",
        "▁o",
        "sika",
        "pa",
        "▁j",
        "ọ",
        "l",
        "ọ",
        "f",
        "▁n",
        "'",
        "eti",
        "ti",
        "▁Na",
        "ị",
        "jir",
        "ịa",
        "▁na",
        "▁Ghana",
        "▁ab",
        "ụ",
        "r",
        "ụ",
        "la",
        "▁ak",
        "ụ",
        "k",
        "ọ"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.583
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 34,
      "token_splits": [
        "▁As",
        "ọ",
        "mp",
        "ị",
        "▁os",
        "ik",
        "apa",
        "▁j",
        "ọ",
        "l",
        "ọ",
        "f",
        "▁n",
        "'",
        "et",
        "iti",
        "▁Na",
        "ị",
        "j",
        "ir",
        "ị",
        "a",
        "▁na",
        "▁Gh",
        "ana",
        "▁ab",
        "ụ",
        "r",
        "ụ",
        "la",
        "▁ak",
        "ụ",
        "k",
        "ọ"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.429
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 32,
      "token_splits": [
        "As",
        "##ọ",
        "##mp",
        "##ị",
        "osi",
        "##kap",
        "##a",
        "j",
        "##ọ",
        "##l",
        "##ọ",
        "##f",
        "n",
        "'",
        "et",
        "##iti",
        "Na",
        "##ị",
        "##jir",
        "##ị",
        "##a",
        "na",
        "Ghana",
        "ab",
        "##ụ",
        "##r",
        "##ụ",
        "##la",
        "ak",
        "##ụ",
        "##k",
        "##ọ"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Asọmpị osikapa jọlọf n'etiti Naịjirịa na Ghana abụrụla akụkọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 32,
      "token_splits": [
        "▁As",
        "ọ",
        "mp",
        "ị",
        "▁os",
        "i",
        "kapa",
        "▁j",
        "ọ",
        "l",
        "ọ",
        "f",
        "▁n",
        "'",
        "et",
        "iti",
        "▁Na",
        "ị",
        "jir",
        "ị",
        "a",
        "▁na",
        "▁Ghana",
        "▁ab",
        "ụ",
        "r",
        "ụ",
        "la",
        "▁ak",
        "ụ",
        "k",
        "ọ"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.462
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "De",
        " j",
        "ollo",
        "f",
        " rice",
        " fight",
        " between",
        " Nigeria",
        " and",
        " Ghana",
        " na",
        " legendary",
        " something"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "De",
        " j",
        "ollo",
        "f",
        " rice",
        " fight",
        " between",
        " Nigeria",
        " and",
        " Ghana",
        " na",
        " legendary",
        " something"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "De",
        "Ġj",
        "ollo",
        "f",
        "Ġrice",
        "Ġfight",
        "Ġbetween",
        "ĠNigeria",
        "Ġand",
        "ĠGhana",
        "Ġna",
        "Ġlegendary",
        "Ġsomething"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "De",
        "▁jo",
        "ll",
        "of",
        "▁rice",
        "▁fight",
        "▁between",
        "▁Nigeria",
        "▁and",
        "▁Ghana",
        "▁na",
        "▁legendary",
        "▁something"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁De",
        "▁j",
        "ol",
        "lo",
        "f",
        "▁rice",
        "▁fight",
        "▁between",
        "▁Nigeria",
        "▁and",
        "▁Gh",
        "ana",
        "▁na",
        "▁legendary",
        "▁something"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.071
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "De",
        "jo",
        "##llo",
        "##f",
        "rice",
        "fight",
        "between",
        "Nigeria",
        "and",
        "Ghana",
        "na",
        "legendary",
        "something"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 48,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De jollof rice fight between Nigeria and Ghana na legendary something",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁De",
        "▁jo",
        "llo",
        "f",
        "▁rice",
        "▁fight",
        "▁between",
        "▁Nigeria",
        "▁and",
        "▁Ghana",
        "▁na",
        "▁legenda",
        "ry",
        "▁something"
      ],
      "english_equivalent": "The jollof rice competition between Nigeria and Ghana is legendary",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.077
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Mobile",
        " money",
        " and",
        " fint",
        "ech",
        " are",
        " revolution",
        "izing",
        " banking",
        " for",
        " the",
        " un",
        "bank",
        "ed",
        " population"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 14,
      "token_splits": [
        "Mobile",
        " money",
        " and",
        " fintech",
        " are",
        " revolution",
        "izing",
        " banking",
        " for",
        " the",
        " un",
        "bank",
        "ed",
        " population"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "Mobile",
        "Ġmoney",
        "Ġand",
        "Ġfint",
        "ech",
        "Ġare",
        "Ġrevolution",
        "izing",
        "Ġbanking",
        "Ġfor",
        "Ġthe",
        "Ġun",
        "bank",
        "ed",
        "Ġpopulation"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 14,
      "token_splits": [
        "Mobile",
        "▁money",
        "▁and",
        "▁fintech",
        "▁are",
        "▁revolution",
        "izing",
        "▁banking",
        "▁for",
        "▁the",
        "▁un",
        "bank",
        "ed",
        "▁population"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁Mobile",
        "▁money",
        "▁and",
        "▁fin",
        "tech",
        "▁are",
        "▁revolution",
        "izing",
        "▁banking",
        "▁for",
        "▁the",
        "▁un",
        "bank",
        "ed",
        "▁population"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "Mobile",
        "money",
        "and",
        "fin",
        "##tech",
        "are",
        "revolution",
        "##izing",
        "banking",
        "for",
        "the",
        "un",
        "##bank",
        "##ed",
        "population"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Mobile",
        "▁money",
        "▁and",
        "▁fin",
        "tech",
        "▁are",
        "▁revolution",
        "izing",
        "▁bank",
        "ing",
        "▁for",
        "▁the",
        "▁un",
        "bank",
        "ed",
        "▁population"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 72,
      "token_splits": [
        "O",
        "w",
        "ó",
        " or",
        "í",
        " f",
        "ó",
        "ò",
        "n",
        "ù",
        " à",
        "ti",
        " f",
        "í",
        "nt",
        "�",
        "�",
        "́",
        "k",
        "ì",
        " ",
        "ń",
        " �",
        "�",
        "�",
        "e",
        " à",
        "y",
        "í",
        "pad",
        "à",
        " �",
        "�",
        "�",
        "�",
        "�",
        "ò",
        "w",
        "ò",
        " il",
        "é",
        " �",
        "�",
        "f",
        "ow",
        "ó",
        "p",
        "am",
        "ọ",
        "́",
        " f",
        "ún",
        " à",
        "w",
        "ọn",
        " tí",
        " k",
        "ò",
        " n",
        "í",
        " à",
        "k",
        "á",
        "�",
        "�",
        "t",
        "ì",
        " b",
        "á",
        "ń",
        "k",
        "ì"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 4.8
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 49,
      "token_splits": [
        "Ow",
        "ó",
        " or",
        "í",
        " fó",
        "òn",
        "ù",
        " àti",
        " fí",
        "nt",
        "ẹ",
        "́",
        "k",
        "ì",
        " ń",
        " ṣe",
        " à",
        "y",
        "í",
        "pad",
        "à",
        " ì",
        "ṣ",
        "ò",
        "w",
        "ò",
        " il",
        "é",
        " ì",
        "f",
        "ow",
        "óp",
        "amọ",
        "́",
        " fún",
        " àwọn",
        " tí",
        " kò",
        " ní",
        " à",
        "ká",
        "�",
        "�",
        "t",
        "ì",
        " bá",
        "ń",
        "k",
        "ì"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 67,
      "token_splits": [
        "O",
        "w",
        "Ã³",
        "Ġor",
        "ÃŃ",
        "Ġf",
        "Ã³",
        "Ã²n",
        "Ã¹",
        "ĠÃł",
        "ti",
        "Ġf",
        "ÃŃ",
        "nt",
        "áº¹",
        "Ìģ",
        "k",
        "Ã¬",
        "Ġ",
        "ÅĦ",
        "Ġá",
        "¹",
        "£",
        "e",
        "ĠÃł",
        "y",
        "ÃŃ",
        "pad",
        "Ãł",
        "ĠÃ",
        "¬",
        "á¹",
        "£",
        "Ã²",
        "w",
        "Ã²",
        "Ġil",
        "Ã©",
        "ĠÃ",
        "¬",
        "f",
        "ow",
        "Ã³",
        "p",
        "am",
        "á»į",
        "Ìģ",
        "Ġf",
        "Ãºn",
        "ĠÃł",
        "w",
        "á»įn",
        "ĠtÃŃ",
        "Ġk",
        "Ã²",
        "ĠnÃŃ",
        "ĠÃł",
        "kÃ¡",
        "Ç",
        "¹",
        "t",
        "Ã¬",
        "Ġb",
        "Ã¡",
        "ÅĦ",
        "k",
        "Ã¬"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 4.467
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 58,
      "token_splits": [
        "Ow",
        "ó",
        "▁or",
        "í",
        "▁fó",
        "òn",
        "ù",
        "▁à",
        "ti",
        "▁f",
        "í",
        "nt",
        "ẹ",
        "́",
        "k",
        "ì",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁à",
        "y",
        "í",
        "pad",
        "à",
        "▁ì",
        "ṣ",
        "ò",
        "w",
        "ò",
        "▁il",
        "é",
        "▁ì",
        "f",
        "ow",
        "ó",
        "pam",
        "ọ",
        "́",
        "▁f",
        "ún",
        "▁à",
        "w",
        "ọn",
        "▁tí",
        "▁k",
        "ò",
        "▁ní",
        "▁à",
        "ká",
        "ǹ",
        "t",
        "ì",
        "▁bá",
        "ń",
        "k",
        "ì"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 4.143
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 72,
      "token_splits": [
        "▁O",
        "w",
        "ó",
        "▁or",
        "í",
        "▁f",
        "ó",
        "ò",
        "n",
        "ù",
        "▁à",
        "ti",
        "▁f",
        "í",
        "nt",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "k",
        "ì",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁à",
        "y",
        "í",
        "pad",
        "à",
        "▁",
        "ì",
        "ṣ",
        "ò",
        "w",
        "ò",
        "▁il",
        "é",
        "▁",
        "ì",
        "f",
        "ow",
        "ó",
        "p",
        "am",
        "ọ",
        "́",
        "▁f",
        "ún",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁t",
        "í",
        "▁k",
        "ò",
        "▁n",
        "í",
        "▁à",
        "k",
        "á",
        "<0xC7>",
        "<0xB9>",
        "t",
        "ì",
        "▁b",
        "á",
        "ń",
        "k",
        "ì"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 4.8
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 53,
      "token_splits": [
        "O",
        "##w",
        "##ó",
        "or",
        "##í",
        "f",
        "##ó",
        "##òn",
        "##ù",
        "àti",
        "f",
        "##ín",
        "##t",
        "##ẹ́",
        "##k",
        "##ì",
        "ń",
        "ṣ",
        "##e",
        "à",
        "##y",
        "##í",
        "##pad",
        "##à",
        "ì",
        "##ṣ",
        "##ò",
        "##w",
        "##ò",
        "il",
        "##é",
        "ì",
        "##fo",
        "##w",
        "##ó",
        "##pa",
        "##m",
        "##ọ́",
        "f",
        "##ún",
        "àwọn",
        "tí",
        "kò",
        "ní",
        "à",
        "##ká",
        "##ǹ",
        "##t",
        "##ì",
        "bá",
        "##ń",
        "##k",
        "##ì"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.533
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Owó orí fóònù àti fíntẹ́kì ń ṣe àyípadà ìṣòwò ilé ìfowópamọ́ fún àwọn tí kò ní àkáǹtì báńkì",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 62,
      "token_splits": [
        "▁O",
        "w",
        "ó",
        "▁or",
        "í",
        "▁fó",
        "ò",
        "n",
        "ù",
        "▁à",
        "ti",
        "▁fí",
        "nt",
        "ẹ",
        "́",
        "k",
        "ì",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁à",
        "y",
        "í",
        "pad",
        "à",
        "▁",
        "ì",
        "ṣ",
        "ò",
        "w",
        "ò",
        "▁i",
        "lé",
        "▁",
        "ì",
        "fo",
        "w",
        "ó",
        "pam",
        "ọ",
        "́",
        "▁fú",
        "n",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁tí",
        "▁k",
        "ò",
        "▁ní",
        "▁à",
        "ká",
        "<unk>",
        "t",
        "ì",
        "▁bá",
        "ń",
        "k",
        "ì"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 16,
      "token_ratio_vs_english": 3.875
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 36,
      "token_splits": [
        "K",
        "u",
        "�",
        "�",
        "in",
        " way",
        "ar",
        " hann",
        "u",
        " da",
        " fint",
        "ech",
        " sun",
        "a",
        " j",
        "uy",
        "in",
        " j",
        "uya",
        " hal",
        "in",
        " bank",
        "i",
        " ga",
        " wa",
        "�",
        "�",
        "anda",
        " ba",
        " su",
        " da",
        " as",
        "us",
        "un",
        " bank",
        "i"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 28,
      "token_splits": [
        "Ku",
        "ɗ",
        "in",
        " way",
        "ar",
        " hannu",
        " da",
        " fintech",
        " suna",
        " j",
        "uy",
        "in",
        " ju",
        "ya",
        " hal",
        "in",
        " bank",
        "i",
        " ga",
        " waɗ",
        "anda",
        " ba",
        " su",
        " da",
        " as",
        "usun",
        " bank",
        "i"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 36,
      "token_splits": [
        "K",
        "u",
        "É",
        "Ĺ",
        "in",
        "Ġway",
        "ar",
        "Ġhann",
        "u",
        "Ġda",
        "Ġfint",
        "ech",
        "Ġsun",
        "a",
        "Ġj",
        "uy",
        "in",
        "Ġj",
        "uya",
        "Ġhal",
        "in",
        "Ġbank",
        "i",
        "Ġga",
        "Ġwa",
        "É",
        "Ĺ",
        "anda",
        "Ġba",
        "Ġsu",
        "Ġda",
        "Ġas",
        "us",
        "un",
        "Ġbank",
        "i"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 29,
      "token_splits": [
        "Ku",
        "ɗ",
        "in",
        "▁way",
        "ar",
        "▁hann",
        "u",
        "▁da",
        "▁fintech",
        "▁suna",
        "▁ju",
        "yin",
        "▁ju",
        "ya",
        "▁hal",
        "in",
        "▁bank",
        "i",
        "▁ga",
        "▁wa",
        "ɗ",
        "anda",
        "▁ba",
        "▁su",
        "▁da",
        "▁asus",
        "un",
        "▁bank",
        "i"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.071
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 37,
      "token_splits": [
        "▁Ku",
        "<0xC9>",
        "<0x97>",
        "in",
        "▁way",
        "ar",
        "▁h",
        "ann",
        "u",
        "▁da",
        "▁fin",
        "tech",
        "▁sun",
        "a",
        "▁j",
        "uy",
        "in",
        "▁j",
        "uy",
        "a",
        "▁hal",
        "in",
        "▁bank",
        "i",
        "▁ga",
        "▁wa",
        "<0xC9>",
        "<0x97>",
        "anda",
        "▁ba",
        "▁su",
        "▁da",
        "▁as",
        "us",
        "un",
        "▁bank",
        "i"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.467
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 27,
      "token_splits": [
        "[UNK]",
        "way",
        "##ar",
        "hann",
        "##u",
        "da",
        "fin",
        "##tech",
        "sun",
        "##a",
        "ju",
        "##yin",
        "ju",
        "##ya",
        "hali",
        "##n",
        "bank",
        "##i",
        "ga",
        "[UNK]",
        "ba",
        "su",
        "da",
        "asus",
        "##un",
        "bank",
        "##i"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.8
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kuɗin wayar hannu da fintech suna juyin juya halin banki ga waɗanda ba su da asusun banki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 28,
      "token_splits": [
        "▁Ku",
        "ɗ",
        "in",
        "▁way",
        "ar",
        "▁hann",
        "u",
        "▁da",
        "▁fin",
        "tech",
        "▁suna",
        "▁ju",
        "yin",
        "▁ju",
        "ya",
        "▁hal",
        "in",
        "▁banki",
        "▁ga",
        "▁wa",
        "ɗ",
        "anda",
        "▁ba",
        "▁su",
        "▁da",
        "▁asu",
        "sun",
        "▁banki"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 49,
      "token_splits": [
        "E",
        "go",
        " ek",
        "went",
        "ị",
        " na",
        " fint",
        "ech",
        " na",
        "-",
        "eme",
        " mg",
        "ban",
        "we",
        " n",
        "'",
        "ụ",
        "z",
        "ọ",
        " ",
        "ụ",
        "l",
        "ọ",
        " ak",
        "ụ",
        " na",
        "-ar",
        "ụ",
        " ",
        "ọ",
        "r",
        "ụ",
        " maka",
        " nd",
        "ị",
        " n",
        "weg",
        "h",
        "ị",
        " aka",
        "ụ",
        "nt",
        "ụ",
        " ",
        "ụ",
        "l",
        "ọ",
        " ak",
        "ụ"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.267
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 27,
      "token_splits": [
        "E",
        "go",
        " ek",
        "went",
        "ị",
        " na",
        " fintech",
        " na",
        "-eme",
        " mgbanwe",
        " n",
        "'ụzọ",
        " ụlọ",
        " akụ",
        " na",
        "-arụ",
        " ọrụ",
        " maka",
        " ndị",
        " n",
        "weghị",
        " aka",
        "ụ",
        "nt",
        "ụ",
        " ụlọ",
        " akụ"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.929
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 49,
      "token_splits": [
        "E",
        "go",
        "Ġek",
        "went",
        "á»ĭ",
        "Ġna",
        "Ġfint",
        "ech",
        "Ġna",
        "-",
        "eme",
        "Ġmg",
        "ban",
        "we",
        "Ġn",
        "'",
        "á»¥",
        "z",
        "á»į",
        "Ġ",
        "á»¥",
        "l",
        "á»į",
        "Ġak",
        "á»¥",
        "Ġna",
        "-ar",
        "á»¥",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        "Ġmaka",
        "Ġnd",
        "á»ĭ",
        "Ġn",
        "weg",
        "h",
        "á»ĭ",
        "Ġaka",
        "á»¥",
        "nt",
        "á»¥",
        "Ġ",
        "á»¥",
        "l",
        "á»į",
        "Ġak",
        "á»¥"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.267
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 47,
      "token_splits": [
        "Ego",
        "▁ek",
        "went",
        "ị",
        "▁na",
        "▁fintech",
        "▁na",
        "-",
        "eme",
        "▁mg",
        "ban",
        "we",
        "▁n",
        "'",
        "ụ",
        "z",
        "ọ",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ",
        "▁na",
        "-",
        "ar",
        "ụ",
        "▁ọ",
        "r",
        "ụ",
        "▁maka",
        "▁nd",
        "ị",
        "▁n",
        "we",
        "gh",
        "ị",
        "▁aka",
        "ụ",
        "nt",
        "ụ",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 3.357
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 53,
      "token_splits": [
        "▁E",
        "go",
        "▁e",
        "kw",
        "ent",
        "ị",
        "▁na",
        "▁fin",
        "tech",
        "▁na",
        "-",
        "eme",
        "▁mg",
        "ban",
        "we",
        "▁n",
        "'",
        "ụ",
        "z",
        "ọ",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ",
        "▁na",
        "-",
        "ar",
        "ụ",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁m",
        "aka",
        "▁nd",
        "ị",
        "▁n",
        "we",
        "gh",
        "ị",
        "▁a",
        "ka",
        "ụ",
        "nt",
        "ụ",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.533
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 47,
      "token_splits": [
        "Ego",
        "ek",
        "##went",
        "##ị",
        "na",
        "fin",
        "##tech",
        "na",
        "-",
        "em",
        "##e",
        "mg",
        "##ban",
        "##we",
        "n",
        "'",
        "ụ",
        "##z",
        "##ọ",
        "ụ",
        "##l",
        "##ọ",
        "ak",
        "##ụ",
        "na",
        "-",
        "ar",
        "##ụ",
        "ọ",
        "##r",
        "##ụ",
        "maka",
        "n",
        "##d",
        "##ị",
        "n",
        "##weg",
        "##hị",
        "aka",
        "##ụ",
        "##nt",
        "##ụ",
        "ụ",
        "##l",
        "##ọ",
        "ak",
        "##ụ"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.133
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ego ekwentị na fintech na-eme mgbanwe n'ụzọ ụlọ akụ na-arụ ọrụ maka ndị nweghị akaụntụ ụlọ akụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 50,
      "token_splits": [
        "▁Ego",
        "▁ek",
        "went",
        "ị",
        "▁na",
        "▁fin",
        "tech",
        "▁na",
        "-",
        "eme",
        "▁mg",
        "ban",
        "we",
        "▁n",
        "'",
        "ụ",
        "z",
        "ọ",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ",
        "▁na",
        "-",
        "ar",
        "ụ",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁maka",
        "▁",
        "nd",
        "ị",
        "▁n",
        "weg",
        "h",
        "ị",
        "▁aka",
        "ụ",
        "nt",
        "ụ",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 16,
      "token_ratio_vs_english": 3.125
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "Mobile",
        " money",
        " and",
        " fint",
        "ech",
        " de",
        "y",
        " change",
        " how",
        " banking",
        " de",
        "y",
        " work",
        " for",
        " people",
        " we",
        "y",
        " no",
        " get",
        " bank",
        " account"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 18,
      "token_splits": [
        "Mobile",
        " money",
        " and",
        " fintech",
        " dey",
        " change",
        " how",
        " banking",
        " dey",
        " work",
        " for",
        " people",
        " we",
        "y",
        " no",
        " get",
        " bank",
        " account"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 21,
      "token_splits": [
        "Mobile",
        "Ġmoney",
        "Ġand",
        "Ġfint",
        "ech",
        "Ġde",
        "y",
        "Ġchange",
        "Ġhow",
        "Ġbanking",
        "Ġde",
        "y",
        "Ġwork",
        "Ġfor",
        "Ġpeople",
        "Ġwe",
        "y",
        "Ġno",
        "Ġget",
        "Ġbank",
        "Ġaccount"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 17,
      "token_splits": [
        "Mobile",
        "▁money",
        "▁and",
        "▁fintech",
        "▁dey",
        "▁change",
        "▁how",
        "▁banking",
        "▁dey",
        "▁work",
        "▁for",
        "▁people",
        "▁wey",
        "▁no",
        "▁get",
        "▁bank",
        "▁account"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.214
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 21,
      "token_splits": [
        "▁Mobile",
        "▁money",
        "▁and",
        "▁fin",
        "tech",
        "▁de",
        "y",
        "▁change",
        "▁how",
        "▁banking",
        "▁de",
        "y",
        "▁work",
        "▁for",
        "▁people",
        "▁we",
        "y",
        "▁no",
        "▁get",
        "▁bank",
        "▁account"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "Mobile",
        "money",
        "and",
        "fin",
        "##tech",
        "de",
        "##y",
        "change",
        "how",
        "banking",
        "de",
        "##y",
        "work",
        "for",
        "people",
        "we",
        "##y",
        "no",
        "get",
        "bank",
        "account"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 49,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Mobile money and fintech dey change how banking dey work for people wey no get bank account",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 22,
      "token_splits": [
        "▁Mobile",
        "▁money",
        "▁and",
        "▁fin",
        "tech",
        "▁de",
        "y",
        "▁change",
        "▁how",
        "▁bank",
        "ing",
        "▁de",
        "y",
        "▁work",
        "▁for",
        "▁people",
        "▁we",
        "y",
        "▁no",
        "▁get",
        "▁bank",
        "▁account"
      ],
      "english_equivalent": "Mobile money and fintech are revolutionizing banking for the unbanked population",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.375
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "N",
        "ollywood",
        " produces",
        " more",
        " films",
        " annually",
        " than",
        " Hollywood",
        ",",
        " making",
        " it",
        " a",
        " global",
        " cultural",
        " force"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 16,
      "token_splits": [
        "N",
        "ol",
        "lywood",
        " produces",
        " more",
        " films",
        " annually",
        " than",
        " Hollywood",
        ",",
        " making",
        " it",
        " a",
        " global",
        " cultural",
        " force"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "N",
        "ollywood",
        "Ġproduces",
        "Ġmore",
        "Ġfilms",
        "Ġannually",
        "Ġthan",
        "ĠHollywood",
        ",",
        "Ġmaking",
        "Ġit",
        "Ġa",
        "Ġglobal",
        "Ġcultural",
        "Ġforce"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 15,
      "token_splits": [
        "No",
        "llywood",
        "▁produces",
        "▁more",
        "▁films",
        "▁annually",
        "▁than",
        "▁Hollywood",
        ",",
        "▁making",
        "▁it",
        "▁a",
        "▁global",
        "▁cultural",
        "▁force"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 16,
      "token_splits": [
        "▁N",
        "olly",
        "wood",
        "▁produces",
        "▁more",
        "▁films",
        "▁annually",
        "▁than",
        "▁Hollywood",
        ",",
        "▁making",
        "▁it",
        "▁a",
        "▁global",
        "▁cultural",
        "▁force"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 16,
      "token_splits": [
        "No",
        "##lly",
        "##wood",
        "produces",
        "more",
        "films",
        "annually",
        "than",
        "Hollywood",
        ",",
        "making",
        "it",
        "a",
        "global",
        "cultural",
        "force"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 17,
      "token_splits": [
        "▁N",
        "ollywood",
        "▁produce",
        "s",
        "▁more",
        "▁films",
        "▁annual",
        "ly",
        "▁than",
        "▁Hollywood",
        ",",
        "▁making",
        "▁it",
        "▁a",
        "▁global",
        "▁cultural",
        "▁force"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 61,
      "token_splits": [
        "N",
        "ollywood",
        " má",
        "a",
        " ",
        "ń",
        " �",
        "�",
        "�",
        "e",
        " f",
        "í",
        "ì",
        "m",
        "ù",
        " t",
        "ó",
        " p",
        "ọ",
        "̀",
        " ju",
        " Hollywood",
        " l",
        "ọ",
        " l",
        "ọ",
        "́",
        "d",
        "ọ",
        "ọ",
        "d",
        "ún",
        ",",
        " è",
        "y",
        "í",
        " s",
        "ì",
        " ti",
        " j",
        "�",
        "�",
        "́",
        " k",
        "í",
        " ó",
        " di",
        " ag",
        "b",
        "á",
        "ra",
        " à",
        "�",
        "�",
        "�",
        "à",
        " à",
        "gb",
        "á",
        "y",
        "é"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 4.067
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 43,
      "token_splits": [
        "N",
        "ol",
        "lywood",
        " má",
        "a",
        " ń",
        " ṣe",
        " fí",
        "ìm",
        "ù",
        " tó",
        " pọ",
        "̀",
        " ju",
        " Hollywood",
        " lọ",
        " lọ",
        "́",
        "d",
        "ọ",
        "ọd",
        "ún",
        ",",
        " è",
        "y",
        "í",
        " sì",
        " ti",
        " jẹ",
        "́",
        " kí",
        " ó",
        " di",
        " agb",
        "ára",
        " à",
        "ṣ",
        "à",
        " à",
        "gb",
        "á",
        "y",
        "é"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 2.688
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 57,
      "token_splits": [
        "N",
        "ollywood",
        "ĠmÃ¡",
        "a",
        "Ġ",
        "ÅĦ",
        "Ġá",
        "¹",
        "£",
        "e",
        "Ġf",
        "ÃŃ",
        "Ã¬m",
        "Ã¹",
        "Ġt",
        "Ã³",
        "Ġp",
        "á»į",
        "ÌĢ",
        "Ġju",
        "ĠHollywood",
        "Ġl",
        "á»į",
        "Ġl",
        "á»į",
        "Ìģ",
        "d",
        "á»į",
        "á»į",
        "d",
        "Ãºn",
        ",",
        "ĠÃ¨",
        "y",
        "ÃŃ",
        "Ġs",
        "Ã¬",
        "Ġti",
        "Ġj",
        "áº¹",
        "Ìģ",
        "Ġk",
        "ÃŃ",
        "ĠÃ³",
        "Ġdi",
        "Ġag",
        "b",
        "Ã¡",
        "ra",
        "ĠÃł",
        "á¹",
        "£",
        "Ãł",
        "ĠÃł",
        "gb",
        "Ã¡y",
        "Ã©"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.8
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 49,
      "token_splits": [
        "No",
        "llywood",
        "▁má",
        "a",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁f",
        "í",
        "ìm",
        "ù",
        "▁tó",
        "▁p",
        "ọ",
        "̀",
        "▁ju",
        "▁Hollywood",
        "▁lọ",
        "▁lọ",
        "́",
        "d",
        "ọ",
        "ọ",
        "d",
        "ún",
        ",",
        "▁è",
        "y",
        "í",
        "▁sì",
        "▁ti",
        "▁j",
        "ẹ",
        "́",
        "▁kí",
        "▁ó",
        "▁di",
        "▁ag",
        "b",
        "ára",
        "▁à",
        "ṣ",
        "à",
        "▁à",
        "gb",
        "áy",
        "é"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.267
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 59,
      "token_splits": [
        "▁N",
        "olly",
        "wood",
        "▁má",
        "a",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁f",
        "í",
        "ì",
        "m",
        "ù",
        "▁t",
        "ó",
        "▁p",
        "ọ",
        "̀",
        "▁ju",
        "▁Hollywood",
        "▁l",
        "ọ",
        "▁l",
        "ọ",
        "́",
        "d",
        "ọ",
        "ọ",
        "d",
        "ún",
        ",",
        "▁è",
        "y",
        "í",
        "▁s",
        "ì",
        "▁ti",
        "▁j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁k",
        "í",
        "▁ó",
        "▁di",
        "▁ag",
        "b",
        "ára",
        "▁à",
        "ṣ",
        "à",
        "▁à",
        "gb",
        "á",
        "y",
        "é"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 3.688
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 48,
      "token_splits": [
        "No",
        "##lly",
        "##wood",
        "má",
        "##a",
        "ń",
        "ṣ",
        "##e",
        "f",
        "##í",
        "##ìm",
        "##ù",
        "tó",
        "p",
        "##ọ̀",
        "ju",
        "Hollywood",
        "l",
        "##ọ",
        "l",
        "##ọ́",
        "##d",
        "##ọ",
        "##ọ",
        "##dú",
        "##n",
        ",",
        "è",
        "##y",
        "##í",
        "sì",
        "ti",
        "jẹ́",
        "k",
        "##í",
        "ó",
        "di",
        "ag",
        "##bá",
        "##ra",
        "à",
        "##ṣ",
        "##à",
        "à",
        "##g",
        "##bá",
        "##y",
        "##é"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Nollywood máa ń ṣe fíìmù tó pọ̀ ju Hollywood lọ lọ́dọọdún, èyí sì ti jẹ́ kí ó di agbára àṣà àgbáyé",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 52,
      "token_splits": [
        "▁N",
        "ollywood",
        "▁má",
        "a",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁fí",
        "ì",
        "m",
        "ù",
        "▁tó",
        "▁p",
        "ọ",
        "̀",
        "▁ju",
        "▁Hollywood",
        "▁l",
        "ọ",
        "▁l",
        "ọ",
        "́",
        "d",
        "ọ",
        "ọ",
        "dú",
        "n",
        ",",
        "▁è",
        "y",
        "í",
        "▁sì",
        "▁ti",
        "▁j",
        "ẹ",
        "́",
        "▁kí",
        "▁ó",
        "▁di",
        "▁ag",
        "bár",
        "a",
        "▁à",
        "ṣ",
        "à",
        "▁à",
        "g",
        "bá",
        "y",
        "é"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 17,
      "token_ratio_vs_english": 3.059
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 41,
      "token_splits": [
        "N",
        "ollywood",
        " t",
        "ana",
        " y",
        "in",
        " fin",
        "a",
        "-fin",
        "ai",
        " fi",
        "ye",
        " da",
        " Hollywood",
        " a",
        " k",
        "ow",
        "ace",
        " she",
        "k",
        "ara",
        ",",
        " w",
        "anda",
        " ya",
        " sa",
        " ta",
        " z",
        "ama",
        " ",
        "�",
        "�",
        "ar",
        "fin",
        " al",
        "'",
        "ada",
        " na",
        " d",
        "uni",
        "ya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.733
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 29,
      "token_splits": [
        "N",
        "ol",
        "lywood",
        " tana",
        " yin",
        " fina",
        "-fin",
        "ai",
        " fi",
        "ye",
        " da",
        " Hollywood",
        " a",
        " kow",
        "ace",
        " shekara",
        ",",
        " wanda",
        " ya",
        " sa",
        " ta",
        " zama",
        " ƙar",
        "fin",
        " al",
        "'",
        "ada",
        " na",
        " duniya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.812
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 40,
      "token_splits": [
        "N",
        "ollywood",
        "Ġt",
        "ana",
        "Ġy",
        "in",
        "Ġfin",
        "a",
        "-fin",
        "ai",
        "Ġfi",
        "ye",
        "Ġda",
        "ĠHollywood",
        "Ġa",
        "Ġk",
        "ow",
        "ace",
        "Ġshe",
        "kara",
        ",",
        "Ġw",
        "anda",
        "Ġya",
        "Ġsa",
        "Ġta",
        "Ġz",
        "ama",
        "Ġ",
        "Æ",
        "Ļ",
        "ar",
        "fin",
        "Ġal",
        "'",
        "ada",
        "Ġna",
        "Ġd",
        "uni",
        "ya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 33,
      "token_splits": [
        "No",
        "llywood",
        "▁tana",
        "▁yin",
        "▁fina",
        "-",
        "f",
        "inai",
        "▁fi",
        "ye",
        "▁da",
        "▁Hollywood",
        "▁a",
        "▁kow",
        "ace",
        "▁she",
        "kara",
        ",",
        "▁wanda",
        "▁ya",
        "▁sa",
        "▁ta",
        "▁zama",
        "▁ƙ",
        "ar",
        "fin",
        "▁al",
        "'",
        "ada",
        "▁na",
        "▁d",
        "uni",
        "ya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.2
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 44,
      "token_splits": [
        "▁N",
        "olly",
        "wood",
        "▁t",
        "ana",
        "▁y",
        "in",
        "▁fin",
        "a",
        "-",
        "f",
        "ina",
        "i",
        "▁fi",
        "ye",
        "▁da",
        "▁Hollywood",
        "▁a",
        "▁k",
        "ow",
        "ace",
        "▁she",
        "k",
        "ara",
        ",",
        "▁w",
        "anda",
        "▁ya",
        "▁sa",
        "▁ta",
        "▁z",
        "ama",
        "▁",
        "<0xC6>",
        "<0x99>",
        "ar",
        "fin",
        "▁al",
        "'",
        "ada",
        "▁na",
        "▁d",
        "uni",
        "ya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 2.75
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 36,
      "token_splits": [
        "No",
        "##lly",
        "##wood",
        "tan",
        "##a",
        "yi",
        "##n",
        "fina",
        "-",
        "fina",
        "##i",
        "fi",
        "##ye",
        "da",
        "Hollywood",
        "a",
        "ko",
        "##wa",
        "##ce",
        "shek",
        "##ara",
        ",",
        "wa",
        "##nda",
        "ya",
        "sa",
        "ta",
        "za",
        "##ma",
        "[UNK]",
        "al",
        "'",
        "ada",
        "na",
        "dun",
        "##iya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Nollywood tana yin fina-finai fiye da Hollywood a kowace shekara, wanda ya sa ta zama ƙarfin al'ada na duniya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 32,
      "token_splits": [
        "▁N",
        "ollywood",
        "▁tana",
        "▁yin",
        "▁fina",
        "-",
        "fina",
        "i",
        "▁fi",
        "ye",
        "▁da",
        "▁Hollywood",
        "▁a",
        "▁",
        "kowa",
        "ce",
        "▁shekara",
        ",",
        "▁wanda",
        "▁ya",
        "▁sa",
        "▁ta",
        "▁zama",
        "▁",
        "ƙ",
        "ar",
        "fin",
        "▁al",
        "'",
        "ada",
        "▁na",
        "▁duniya"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.882
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 38,
      "token_splits": [
        "N",
        "ollywood",
        " na",
        "-em",
        "ep",
        "ụ",
        "ta",
        " i",
        "he",
        " nk",
        "iri",
        " kar",
        "ị",
        "a",
        " Hollywood",
        " k",
        "wa",
        " af",
        "ọ",
        ",",
        " n",
        "ke",
        " mere",
        " ka",
        " ",
        "ọ",
        " b",
        "ụ",
        "r",
        "ụ",
        " i",
        "ke",
        " o",
        "men",
        "ala",
        " ",
        "ụ",
        "wa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.533
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 22,
      "token_splits": [
        "N",
        "ol",
        "lywood",
        " na",
        "-em",
        "epụta",
        " ihe",
        " nkiri",
        " karịa",
        " Hollywood",
        " kwa",
        " afọ",
        ",",
        " nke",
        " mere",
        " ka",
        " ọ",
        " bụrụ",
        " ike",
        " omen",
        "ala",
        " ụwa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.375
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 36,
      "token_splits": [
        "N",
        "ollywood",
        "Ġna",
        "-em",
        "ep",
        "á»¥",
        "ta",
        "Ġi",
        "he",
        "Ġnk",
        "iri",
        "Ġkar",
        "á»ĭa",
        "ĠHollywood",
        "Ġk",
        "wa",
        "Ġaf",
        "á»į",
        ",",
        "Ġn",
        "ke",
        "Ġmere",
        "Ġka",
        "Ġ",
        "á»į",
        "Ġbá»¥",
        "r",
        "á»¥",
        "Ġi",
        "ke",
        "Ġo",
        "men",
        "ala",
        "Ġ",
        "á»¥",
        "wa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.4
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 31,
      "token_splits": [
        "No",
        "llywood",
        "▁na",
        "-",
        "eme",
        "p",
        "ụ",
        "ta",
        "▁ihe",
        "▁n",
        "kiri",
        "▁kar",
        "ịa",
        "▁Hollywood",
        "▁kwa",
        "▁af",
        "ọ",
        ",",
        "▁nke",
        "▁mere",
        "▁ka",
        "▁ọ",
        "▁bụ",
        "r",
        "ụ",
        "▁ike",
        "▁omen",
        "ala",
        "▁",
        "ụ",
        "wa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.067
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 41,
      "token_splits": [
        "▁N",
        "olly",
        "wood",
        "▁na",
        "-",
        "em",
        "ep",
        "ụ",
        "ta",
        "▁i",
        "he",
        "▁n",
        "k",
        "iri",
        "▁kar",
        "ị",
        "a",
        "▁Hollywood",
        "▁k",
        "wa",
        "▁af",
        "ọ",
        ",",
        "▁n",
        "ke",
        "▁mere",
        "▁ka",
        "▁",
        "ọ",
        "▁b",
        "ụ",
        "r",
        "ụ",
        "▁i",
        "ke",
        "▁o",
        "men",
        "ala",
        "▁",
        "ụ",
        "wa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 2.562
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 38,
      "token_splits": [
        "No",
        "##lly",
        "##wood",
        "na",
        "-",
        "em",
        "##ep",
        "##ụ",
        "##ta",
        "ih",
        "##e",
        "n",
        "##kir",
        "##i",
        "kar",
        "##ị",
        "##a",
        "Hollywood",
        "kwa",
        "af",
        "##ọ",
        ",",
        "n",
        "##ke",
        "mere",
        "ka",
        "ọ",
        "b",
        "##ụ",
        "##r",
        "##ụ",
        "ik",
        "##e",
        "om",
        "##enal",
        "##a",
        "ụ",
        "##wa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 2.375
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nollywood na-emepụta ihe nkiri karịa Hollywood kwa afọ, nke mere ka ọ bụrụ ike omenala ụwa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 37,
      "token_splits": [
        "▁N",
        "ollywood",
        "▁na",
        "-",
        "em",
        "ep",
        "ụ",
        "ta",
        "▁i",
        "he",
        "▁n",
        "kiri",
        "▁kar",
        "ị",
        "a",
        "▁Hollywood",
        "▁kwa",
        "▁af",
        "ọ",
        ",",
        "▁",
        "nke",
        "▁mere",
        "▁ka",
        "▁",
        "ọ",
        "▁b",
        "ụ",
        "r",
        "ụ",
        "▁i",
        "ke",
        "▁omen",
        "ala",
        "▁",
        "ụ",
        "wa"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 17,
      "token_ratio_vs_english": 2.176
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "ollywood",
        " de",
        "y",
        " make",
        " more",
        " film",
        " pass",
        " Hollywood",
        " every",
        " year",
        ",",
        " dat",
        " one",
        " make",
        " am",
        " become",
        " world",
        " power",
        " for",
        " culture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "ol",
        "lywood",
        " dey",
        " make",
        " more",
        " film",
        " pass",
        " Hollywood",
        " every",
        " year",
        ",",
        " dat",
        " one",
        " make",
        " am",
        " become",
        " world",
        " power",
        " for",
        " culture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.312
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "ollywood",
        "Ġde",
        "y",
        "Ġmake",
        "Ġmore",
        "Ġfilm",
        "Ġpass",
        "ĠHollywood",
        "Ġevery",
        "Ġyear",
        ",",
        "Ġdat",
        "Ġone",
        "Ġmake",
        "Ġam",
        "Ġbecome",
        "Ġworld",
        "Ġpower",
        "Ġfor",
        "Ġculture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 20,
      "token_splits": [
        "No",
        "llywood",
        "▁dey",
        "▁make",
        "▁more",
        "▁film",
        "▁pass",
        "▁Hollywood",
        "▁every",
        "▁year",
        ",",
        "▁dat",
        "▁one",
        "▁make",
        "▁am",
        "▁become",
        "▁world",
        "▁power",
        "▁for",
        "▁culture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁N",
        "olly",
        "wood",
        "▁de",
        "y",
        "▁make",
        "▁more",
        "▁film",
        "▁pass",
        "▁Hollywood",
        "▁every",
        "▁year",
        ",",
        "▁dat",
        "▁one",
        "▁make",
        "▁am",
        "▁become",
        "▁world",
        "▁power",
        "▁for",
        "▁culture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.375
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 22,
      "token_splits": [
        "No",
        "##lly",
        "##wood",
        "de",
        "##y",
        "make",
        "more",
        "film",
        "pass",
        "Hollywood",
        "every",
        "year",
        ",",
        "dat",
        "one",
        "make",
        "am",
        "become",
        "world",
        "power",
        "for",
        "culture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 16,
      "token_ratio_vs_english": 1.375
    },
    {
      "entry_id": 50,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Nollywood dey make more film pass Hollywood every year, dat one make am become world power for culture",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 21,
      "token_splits": [
        "▁N",
        "ollywood",
        "▁de",
        "y",
        "▁make",
        "▁more",
        "▁film",
        "▁pass",
        "▁Hollywood",
        "▁every",
        "▁year",
        ",",
        "▁dat",
        "▁one",
        "▁make",
        "▁am",
        "▁become",
        "▁world",
        "▁power",
        "▁for",
        "▁culture"
      ],
      "english_equivalent": "Nollywood produces more films annually than Hollywood, making it a global cultural force",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.235
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "teacher",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ol",
        "ù",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ol",
        "ù",
        "kọ",
        "́"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "ol",
        "Ã¹",
        "k",
        "á»į",
        "Ìģ"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ol",
        "ù",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁ol",
        "ù",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ol",
        "##ù",
        "##k",
        "##ọ́"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "olùkọ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁ol",
        "ù",
        "k",
        "ọ",
        "́"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "mal",
        "ami"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "mal",
        "ami"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "mal",
        "ami"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "mal",
        "ami"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁mal",
        "ami"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "malam",
        "##i"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "malami",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁malam",
        "i"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ony",
        "e",
        " nk",
        "ụ",
        "zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "onye",
        " nk",
        "ụ",
        "zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "ony",
        "e",
        "Ġnk",
        "á»¥",
        "zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "on",
        "ye",
        "▁nk",
        "ụ",
        "zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁on",
        "ye",
        "▁n",
        "k",
        "ụ",
        "zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "on",
        "##ye",
        "n",
        "##k",
        "##ụ",
        "##zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "onye nkụzi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁on",
        "ye",
        "▁nk",
        "ụ",
        "zi"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 51,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "teacher",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁teacher"
      ],
      "english_equivalent": "teacher",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "doctor",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "on",
        "í",
        "�",
        "�",
        "�",
        "è",
        "g",
        "ù",
        "n"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "on",
        "í",
        "ṣ",
        "èg",
        "ùn"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "on",
        "ÃŃ",
        "á¹",
        "£",
        "Ã¨",
        "g",
        "Ã¹",
        "n"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "on",
        "í",
        "ṣ",
        "è",
        "g",
        "ùn"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁on",
        "í",
        "ṣ",
        "è",
        "g",
        "ù",
        "n"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "on",
        "##í",
        "##ṣ",
        "##è",
        "##g",
        "##ùn"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oníṣègùn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁o",
        "ní",
        "ṣ",
        "è",
        "g",
        "ù",
        "n"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "lik",
        "ita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "lik",
        "ita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "lik",
        "ita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "lik",
        "ita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁lik",
        "ita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "lik",
        "##ita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "likita",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁li",
        "kita"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "d",
        "ib",
        "ị",
        "a"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "dib",
        "ịa"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "d",
        "ib",
        "á»ĭa"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "dib",
        "ịa"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁d",
        "ib",
        "ị",
        "a"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "di",
        "##b",
        "##ị",
        "##a"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dibịa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁dib",
        "ị",
        "a"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 52,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "doctor",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁doctor"
      ],
      "english_equivalent": "doctor",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "friend",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁friend"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "ọ",
        "̀",
        "r",
        "�",
        "�",
        "́"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "̀",
        "rẹ",
        "́"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "á»į",
        "ÌĢ",
        "r",
        "áº¹",
        "Ìģ"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "̀",
        "r",
        "ẹ",
        "́"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁",
        "ọ",
        "̀",
        "r",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "##̀",
        "##r",
        "##ẹ́"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀rẹ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ọ",
        "̀",
        "r",
        "ẹ",
        "́"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "oki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "oki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "oki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "oki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁ab",
        "oki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ab",
        "##oki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "aboki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁abo",
        "ki"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "eny",
        "i"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "en",
        "yi"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "eny",
        "i"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "en",
        "yi"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁en",
        "y",
        "i"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "en",
        "##yi"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "enyi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁en",
        "yi"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "addy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "addy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "addy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "p",
        "addy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁p",
        "addy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "pad",
        "##dy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 53,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "paddy",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁pa",
        "ddy"
      ],
      "english_equivalent": "friend",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "sun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "o",
        "ò",
        "r",
        "ù",
        "n"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "o",
        "òr",
        "ùn"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "o",
        "Ã²",
        "r",
        "Ã¹",
        "n"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "o",
        "ò",
        "r",
        "ùn"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁o",
        "ò",
        "r",
        "ù",
        "n"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "o",
        "##ò",
        "##r",
        "##ùn"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "oòrùn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁o",
        "òr",
        "ù",
        "n"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "r",
        "ana"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "rana"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "rana"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "rana"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁r",
        "ana"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ran",
        "##a"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rana",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁rana"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "any",
        "an",
        "w",
        "ụ"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "any",
        "an",
        "wụ"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "any",
        "an",
        "w",
        "á»¥"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "anyan",
        "w",
        "ụ"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁any",
        "an",
        "w",
        "ụ"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "any",
        "##an",
        "##w",
        "##ụ"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anyanwụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁any",
        "an",
        "w",
        "ụ"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 54,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁sun"
      ],
      "english_equivalent": "sun",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "eng",
      "language_name": "English",
      "text": "rain",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ò",
        "j",
        "ò"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ò",
        "j",
        "ò"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Ã²",
        "j",
        "Ã²"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ò",
        "j",
        "ò"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ò",
        "j",
        "ò"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ò",
        "##j",
        "##ò"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "òjò",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ò",
        "j",
        "ò"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ru",
        "wan",
        " sama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ru",
        "wan",
        " sama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ru",
        "wan",
        "Ġsama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ru",
        "wan",
        "▁sama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁ru",
        "wan",
        "▁s",
        "ama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ru",
        "##wan",
        "sama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ruwan sama",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ruwa",
        "n",
        "▁sama"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "mm",
        "ị",
        "r",
        "ị",
        " ",
        "ọ",
        "z",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "mm",
        "ịrị",
        " ọ",
        "zụ",
        "zọ"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "mm",
        "á»ĭ",
        "r",
        "á»ĭ",
        "Ġ",
        "á»į",
        "z",
        "á»¥",
        "z",
        "á»į"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "mm",
        "ị",
        "r",
        "ị",
        "▁ọ",
        "z",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁mm",
        "ị",
        "r",
        "ị",
        "▁",
        "ọ",
        "z",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "mm",
        "##ị",
        "##r",
        "##ị",
        "ọ",
        "##z",
        "##ụ",
        "##z",
        "##ọ"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmịrị ọzụzọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁mm",
        "ị",
        "r",
        "ị",
        "▁",
        "ọ",
        "z",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 55,
      "category": "common_nouns",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "rain",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁rain"
      ],
      "english_equivalent": "rain",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "head",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "or",
        "í"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "or",
        "í"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "or",
        "ÃŃ"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "or",
        "í"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁or",
        "í"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "or",
        "##í"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "orí",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁or",
        "í"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "k",
        "ai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "kai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "k",
        "ai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "kai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁k",
        "ai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "kai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "kai",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁kai"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "isi"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "isi"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "isi"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "isi"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁is",
        "i"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "is",
        "##i"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "isi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁isi"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 56,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "head",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁head"
      ],
      "english_equivalent": "head",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "hand",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ọwọ",
        "́"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "á»į",
        "w",
        "á»į",
        "Ìģ"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "ọ",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "##w",
        "##ọ́"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọwọ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "ọ",
        "w",
        "ọ",
        "́"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "h",
        "ann",
        "u"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "h",
        "annu"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "h",
        "ann",
        "u"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "h",
        "annu"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁h",
        "ann",
        "u"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "hann",
        "##u"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "hannu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁hann",
        "u"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "aka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "aka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "aka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "aka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁a",
        "ka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "aka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "aka",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁aka"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 57,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hand",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁hand"
      ],
      "english_equivalent": "hand",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "eye",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "oj",
        "ú"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "oj",
        "ú"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "oj",
        "Ãº"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "oj",
        "ú"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁o",
        "j",
        "ú"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "o",
        "##jú"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ojú",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁o",
        "jú"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ido"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ido"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "ido"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "ido"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁id",
        "o"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "ido"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ido",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁i",
        "do"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "anya"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "anya"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "anya"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "anya"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁any",
        "a"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "any",
        "##a"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "anya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁a",
        "nya"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 58,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "eye",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁eye"
      ],
      "english_equivalent": "eye",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "st",
        "om",
        "ach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "st",
        "omach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "st",
        "om",
        "ach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "sto",
        "mach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁stomach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "sto",
        "##mach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "stomach",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁stomach"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ik",
        "ù",
        "n"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ik",
        "ùn"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ik",
        "Ã¹",
        "n"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ik",
        "ùn"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁ik",
        "ù",
        "n"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ik",
        "##ùn"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ikùn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ik",
        "ù",
        "n"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "c",
        "iki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "c",
        "iki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "c",
        "iki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ci",
        "ki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁c",
        "iki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ci",
        "##ki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ciki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁ciki"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "af",
        "ọ"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "af",
        "ọ"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "af",
        "á»į"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "af",
        "ọ"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁af",
        "ọ"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "af",
        "##ọ"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "afọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁af",
        "ọ"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "b",
        "elle"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "b",
        "elle"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "b",
        "elle"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "belle"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁bel",
        "le"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "belle"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 59,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "belle",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁belle"
      ],
      "english_equivalent": "stomach",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "eng",
      "language_name": "English",
      "text": "heart",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "k",
        "àn"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "k",
        "àn"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "á»į",
        "k",
        "Ãłn"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "k",
        "àn"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "ọ",
        "k",
        "à",
        "n"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "##k",
        "##àn"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọkàn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ọ",
        "k",
        "àn"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "uci",
        "ya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "uc",
        "iya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "z",
        "uci",
        "ya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "zu",
        "ci",
        "ya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁zu",
        "ci",
        "ya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "zu",
        "##ci",
        "##ya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "hau",
      "language_name": "Hausa",
      "text": "zuciya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁zu",
        "ci",
        "ya"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "obi"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "obi"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "obi"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "obi"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁ob",
        "i"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ob",
        "##i"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁obi"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 60,
      "category": "body_parts",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "heart",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁heart"
      ],
      "english_equivalent": "heart",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "esterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "esterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "esterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "yesterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁yesterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ye",
        "##ster",
        "##day"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "yesterday",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁yesterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "l",
        "án",
        "à",
        "á"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "lán",
        "à",
        "á"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "lÃ¡n",
        "Ãł",
        "Ã¡"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "lán",
        "à",
        "á"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁l",
        "án",
        "à",
        "á"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "lá",
        "##n",
        "##à",
        "##á"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lánàá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁lán",
        "à",
        "á"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ji",
        "ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ji",
        "ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ji",
        "ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ji",
        "ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁ji",
        "ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ji",
        "##ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jiya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ji",
        "ya"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ụ",
        "nya",
        "ah",
        "ụ"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ụ",
        "nya",
        "ah",
        "ụ"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "á»¥",
        "nya",
        "ah",
        "á»¥"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "ụ",
        "nya",
        "ah",
        "ụ"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ụ",
        "ny",
        "a",
        "ah",
        "ụ"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ụ",
        "##nya",
        "##ah",
        "##ụ"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụnyaahụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "ụ",
        "nya",
        "ah",
        "ụ"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "esterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "esterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "esterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "yesterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁yesterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ye",
        "##ster",
        "##day"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 61,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yesterday",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁yesterday"
      ],
      "english_equivalent": "yesterday",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "tom",
        "orrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "tom",
        "orrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "tom",
        "orrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "tomorrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁tomorrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "tom",
        "##or",
        "##row"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "tomorrow",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁tomorrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "̀",
        "la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "̀",
        "la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "á»į",
        "ÌĢ",
        "la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "̀",
        "la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ọ",
        "̀",
        "la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "##̀",
        "##la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọ̀la",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ọ",
        "̀",
        "la"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "obe"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "obe"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "obe"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "go",
        "be"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁go",
        "be"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "go",
        "##be"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gobe",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁go",
        "be"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ech",
        "i"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ech",
        "i"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ech",
        "i"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "echi"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁e",
        "chi"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "e",
        "##chi"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "echi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁e",
        "chi"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "tom",
        "orrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "tom",
        "orrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "tom",
        "orrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "tomorrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁tomorrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "tom",
        "##or",
        "##row"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 62,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "tomorrow",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁tomorrow"
      ],
      "english_equivalent": "tomorrow",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "eng",
      "language_name": "English",
      "text": "today",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "l",
        "ón",
        "ì",
        "í"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "l",
        "ón",
        "ì",
        "í"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "l",
        "Ã³n",
        "Ã¬",
        "ÃŃ"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "l",
        "ón",
        "ì",
        "í"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁l",
        "ón",
        "ì",
        "í"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "l",
        "##ón",
        "##ìí"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "lónìí",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "lón",
        "ì",
        "í"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "au"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "au"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "au"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "au"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁y",
        "au"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ya",
        "##u"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yau",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ya",
        "u"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "a"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "taa"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "a"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "taa"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁t",
        "aa"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ta",
        "##a"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "taa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ta",
        "a"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 63,
      "category": "time",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "today",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁today"
      ],
      "english_equivalent": "today",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "alm",
        " wine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "alm",
        " wine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "alm",
        "Ġwine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "palm",
        "▁wine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁palm",
        "▁wine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "palm",
        "wine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "palm wine",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁palm",
        "▁wine"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "�",
        "�",
        "mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ẹ",
        "mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "áº¹",
        "mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ẹ",
        "mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ẹ",
        "##mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹmu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "ẹ",
        "mu"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "bam",
        "mi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "bam",
        "mi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "bam",
        "mi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "bam",
        "mi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁b",
        "am",
        "mi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ba",
        "##mmi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "bammi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁bam",
        "mi"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "m",
        "man",
        "ya",
        " ng",
        "w",
        "ọ"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "mm",
        "anya",
        " ng",
        "wọ"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "m",
        "man",
        "ya",
        "Ġng",
        "w",
        "á»į"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "m",
        "manya",
        "▁ng",
        "w",
        "ọ"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁mm",
        "anya",
        "▁ng",
        "w",
        "ọ"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "mm",
        "##anya",
        "ng",
        "##w",
        "##ọ"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mmanya ngwọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "mma",
        "nya",
        "▁ng",
        "w",
        "ọ"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "alm",
        "ie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "alm",
        "ie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "p",
        "alm",
        "ie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "pal",
        "mie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁palm",
        "ie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "palm",
        "##ie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 64,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "palmie",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁palm",
        "ie"
      ],
      "english_equivalent": "palm wine",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "pe",
        "pper",
        " soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "pep",
        "per",
        " soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "pe",
        "pper",
        "Ġsoup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "pepper",
        "▁soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁pepper",
        "▁soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "pe",
        "##pper",
        "sou",
        "##p"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "pepper soup",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁pepper",
        "▁soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "ọ",
        "b",
        "�",
        "�",
        "̀",
        " ata"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "b",
        "ẹ",
        "̀",
        " ata"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "á»į",
        "b",
        "áº¹",
        "ÌĢ",
        "Ġata"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "b",
        "ẹ",
        "̀",
        "▁ata"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "ọ",
        "b",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "▁at",
        "a"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "##b",
        "##ẹ̀",
        "ata"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ọbẹ̀ ata",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ọ",
        "b",
        "ẹ",
        "̀",
        "▁ata"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "mi",
        "yan",
        " bark",
        "ono"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "m",
        "iyan",
        " bark",
        "ono"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "mi",
        "yan",
        "Ġbark",
        "ono"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "mi",
        "yan",
        "▁bark",
        "ono"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁mi",
        "yan",
        "▁bark",
        "ono"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "mi",
        "##yan",
        "bar",
        "##kon",
        "##o"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "miyan barkono",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁mi",
        "yan",
        "▁bar",
        "ko",
        "no"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "of",
        "e",
        " o",
        "se"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "of",
        "e",
        " ose"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "of",
        "e",
        "Ġo",
        "se"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ofe",
        "▁ose"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁of",
        "e",
        "▁o",
        "se"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "of",
        "##e",
        "ose"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ofe ose",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁of",
        "e",
        "▁ose"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "pe",
        "pper",
        " soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "pep",
        "per",
        " soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "pe",
        "pper",
        "Ġsoup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "pepper",
        "▁soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁pepper",
        "▁soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "pe",
        "##pper",
        "sou",
        "##p"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 65,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "pepper soup",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁pepper",
        "▁soup"
      ],
      "english_equivalent": "pepper soup",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "am"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "yam"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "am"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "yam"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁y",
        "am"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ya",
        "##m"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "yam",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ya",
        "m"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "i",
        "�",
        "�",
        "�",
        "u"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "i",
        "ṣu"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "i",
        "á¹",
        "£",
        "u"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "i",
        "ṣ",
        "u"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁i",
        "ṣ",
        "u"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "i",
        "##ṣ",
        "##u"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iṣu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁i",
        "ṣ",
        "u"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "oya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "oya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "d",
        "oya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "do",
        "ya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁do",
        "ya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "do",
        "##ya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "doya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁do",
        "ya"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ji",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁ji"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "am"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "yam"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "y",
        "am"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "yam"
      ],
      "english_equivalent": "yam",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁y",
        "am"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ya",
        "##m"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 66,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "yam",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁ya",
        "m"
      ],
      "english_equivalent": "yam",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "c",
        "ass",
        "ava"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "cass",
        "ava"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "c",
        "ass",
        "ava"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "cass",
        "ava"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁c",
        "ass",
        "ava"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "cas",
        "##sa",
        "##va"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "eng",
      "language_name": "English",
      "text": "cassava",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ca",
        "ssa",
        "va"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "�",
        "�",
        "̀",
        "g",
        "�",
        "�",
        "́"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ẹ",
        "̀",
        "gẹ",
        "́"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "áº¹",
        "ÌĢ",
        "g",
        "áº¹",
        "Ìģ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ẹ",
        "̀",
        "g",
        "ẹ",
        "́"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "g",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "ẹ",
        "##̀",
        "##g",
        "##ẹ́"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀gẹ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ẹ",
        "̀",
        "g",
        "ẹ",
        "́"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ro",
        "go"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ro",
        "go"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ro",
        "go"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ro",
        "go"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁ro",
        "go"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ro",
        "##go"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rogo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁rog",
        "o"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ak",
        "pụ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ak",
        "p",
        "á»¥"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ak",
        "##p",
        "##ụ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "akpụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "gar",
        "ri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "gar",
        "ri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "gar",
        "ri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "gar",
        "ri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁gar",
        "ri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "gar",
        "##ri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 67,
      "category": "food_drink",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "garri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "garri"
      ],
      "english_equivalent": "cassava",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġrun"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to run",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "ár",
        "é"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "ár",
        "é"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "Ã¡r",
        "Ã©"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "á",
        "ré"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁s",
        "ár",
        "é"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "##ár",
        "##é"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "sáré",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁sá",
        "ré"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "udu"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "udu"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "g",
        "udu"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "gu",
        "du"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁g",
        "ud",
        "u"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "gud",
        "##u"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "gudu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁",
        "gudu"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "gba",
        "a",
        " ",
        "ọ",
        "s",
        "ọ"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "g",
        "baa",
        " ọsọ"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "gba",
        "a",
        "Ġ",
        "á»į",
        "s",
        "á»į"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "g",
        "baa",
        "▁ọ",
        "s",
        "ọ"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁g",
        "ba",
        "a",
        "▁",
        "ọ",
        "s",
        "ọ"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "g",
        "##ba",
        "##a",
        "ọ",
        "##s",
        "##ọ"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa ọsọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁g",
        "ba",
        "a",
        "▁",
        "ọ",
        "s",
        "ọ"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 68,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "run",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁run"
      ],
      "english_equivalent": "to run",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġsing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to sing",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "ọ",
        "rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "kọ",
        "rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "á»į",
        "rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "ọ",
        "rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁k",
        "ọ",
        "rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "##ọ",
        "##rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọrin",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁k",
        "ọ",
        "rin"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "r",
        "era",
        " wa",
        "�",
        "�",
        "a"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "r",
        "era",
        " wa",
        "ƙ",
        "a"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "r",
        "era",
        "Ġwa",
        "Æ",
        "Ļ",
        "a"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "rera",
        "▁wa",
        "ƙ",
        "a"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁r",
        "era",
        "▁wa",
        "<0xC6>",
        "<0x99>",
        "a"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "re",
        "##ra",
        "[UNK]"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rera waƙa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁re",
        "ra",
        "▁wa",
        "ƙ",
        "a"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "we",
        "e",
        " ab",
        "ụ"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "k",
        "wee",
        " ab",
        "ụ"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "we",
        "e",
        "Ġab",
        "á»¥"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "k",
        "wee",
        "▁ab",
        "ụ"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁k",
        "we",
        "e",
        "▁ab",
        "ụ"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "##we",
        "##e",
        "ab",
        "##ụ"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "kwee abụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁kwe",
        "e",
        "▁ab",
        "ụ"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 69,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "sing",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁sing"
      ],
      "english_equivalent": "to sing",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġcook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "to",
        "co",
        "##ok"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to cook",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "�",
        "�",
        "�",
        "e",
        " o",
        "ún",
        "j",
        "�",
        "�"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ṣe",
        " o",
        "ún",
        "jẹ"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "á¹",
        "£",
        "e",
        "Ġo",
        "Ãºn",
        "j",
        "áº¹"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "ṣ",
        "e",
        "▁o",
        "ún",
        "j",
        "ẹ"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "ṣ",
        "e",
        "▁o",
        "ún",
        "j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "ṣ",
        "##e",
        "o",
        "##ún",
        "##j",
        "##ẹ"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ṣe oúnjẹ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁",
        "ṣ",
        "e",
        "▁o",
        "ún",
        "j",
        "ẹ"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "d",
        "afa",
        " ab",
        "inci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "d",
        "afa",
        " ab",
        "inci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "d",
        "afa",
        "Ġab",
        "inci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "da",
        "fa",
        "▁ab",
        "inci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁da",
        "fa",
        "▁ab",
        "in",
        "ci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "da",
        "##fa",
        "ab",
        "##inci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "dafa abinci",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁da",
        "fa",
        "▁abin",
        "ci"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "sie",
        " n",
        "ri"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "sie",
        " nri"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "sie",
        "Ġn",
        "ri"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "sie",
        "▁n",
        "ri"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁sie",
        "▁n",
        "ri"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "sie",
        "nr",
        "##i"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "sie nri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁sie",
        "▁nr",
        "i"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "co",
        "##ok"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 70,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "cook",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁cook"
      ],
      "english_equivalent": "to cook",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġwrite"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to write",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "ọ",
        "̀",
        "w",
        "é"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "kọ",
        "̀",
        "wé"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "á»į",
        "ÌĢ",
        "w",
        "Ã©"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "k",
        "ọ",
        "̀",
        "w",
        "é"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁k",
        "ọ",
        "̀",
        "w",
        "é"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "k",
        "##ọ̀",
        "##wé"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kọ̀wé",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁k",
        "ọ",
        "̀",
        "wé"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "rub",
        "uta"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "rub",
        "uta"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "rub",
        "uta"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "rub",
        "uta"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁rub",
        "uta"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ru",
        "##but",
        "##a"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "rubuta",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁rub",
        "uta"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "dee",
        " i",
        "he"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "dee",
        " ihe"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "dee",
        "Ġi",
        "he"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "dee",
        "▁ihe"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁de",
        "e",
        "▁i",
        "he"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "dee",
        "ih",
        "##e"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "dee ihe",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁de",
        "e",
        "▁i",
        "he"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 71,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "write",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁write"
      ],
      "english_equivalent": "to write",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "to",
        " dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "Ġdance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "▁dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "to",
        "dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "eng",
      "language_name": "English",
      "text": "to dance",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁to",
        "▁dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "j",
        "ó"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "jó"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "j",
        "Ã³"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "jó"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁j",
        "ó"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "jó"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "jó",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁jó"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        " raw",
        "a"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "yi",
        " rawa"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "Ġraw",
        "a"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "▁ra",
        "wa"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁y",
        "i",
        "▁raw",
        "a"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "yi",
        "raw",
        "##a"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "yi rawa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁yi",
        "▁ra",
        "wa"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "gba",
        "a",
        " eg",
        "wu"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "g",
        "baa",
        " egwu"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "gba",
        "a",
        "Ġeg",
        "wu"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "g",
        "baa",
        "▁eg",
        "wu"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁g",
        "ba",
        "a",
        "▁eg",
        "w",
        "u"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "g",
        "##ba",
        "##a",
        "e",
        "##g",
        "##wu"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "gbaa egwu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁g",
        "ba",
        "a",
        "▁eg",
        "wu"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 72,
      "category": "verbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "dance",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁dance"
      ],
      "english_equivalent": "to dance",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "small",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "k",
        "é",
        "ker",
        "é"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ké",
        "ker",
        "é"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "kÃ©",
        "ker",
        "Ã©"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ké",
        "ker",
        "é"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁k",
        "é",
        "ker",
        "é"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "kékeré"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "kékeré",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁kék",
        "er",
        "é"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "�",
        "�",
        "aram",
        "i"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ƙ",
        "ar",
        "ami"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Æ",
        "Ļ",
        "aram",
        "i"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ƙ",
        "ar",
        "ami"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "<0xC6>",
        "<0x99>",
        "aram",
        "i"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "[UNK]"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "ƙarami",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "ƙ",
        "ar",
        "ami"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ob",
        "ere"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ob",
        "ere"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "ob",
        "ere"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "ob",
        "ere"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁ob",
        "ere"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "ober",
        "##e"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "obere",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁o",
        "bere"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 73,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "small",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁small"
      ],
      "english_equivalent": "small",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "old",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "à",
        "gb",
        "à",
        "lag",
        "b",
        "à"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "à",
        "gb",
        "à",
        "lag",
        "b",
        "à"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "Ãł",
        "gb",
        "Ãł",
        "lag",
        "b",
        "Ãł"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "à",
        "gb",
        "à",
        "lag",
        "b",
        "à"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁à",
        "gb",
        "à",
        "lag",
        "b",
        "à"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "à",
        "##g",
        "##bà",
        "##lag",
        "##bà"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "àgbàlagbà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁à",
        "gb",
        "à",
        "lag",
        "b",
        "à"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "so",
        "ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "so",
        "ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "so",
        "ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "tso",
        "ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁t",
        "so",
        "ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "##so",
        "##ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "tsoho",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁",
        "tso",
        "ho"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "och",
        "ie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "och",
        "ie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "och",
        "ie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "o",
        "chie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁och",
        "ie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "och",
        "##ie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ochie",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁och",
        "ie"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 74,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "old",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁old"
      ],
      "english_equivalent": "old",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "eng",
      "language_name": "English",
      "text": "new",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "unt",
        "un"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "unt",
        "un"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "t",
        "unt",
        "un"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "tu",
        "nt",
        "un"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁t",
        "unt",
        "un"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "tun",
        "##tun"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "tuntun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁tuntu",
        "n"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "ab",
        "on"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "s",
        "abon"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "s",
        "ab",
        "on"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "sab",
        "on"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁sab",
        "on"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "sa",
        "##bon"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "hau",
      "language_name": "Hausa",
      "text": "sabon",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁sabon"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọ",
        "h",
        "ụrụ"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "á»į",
        "h",
        "á»¥",
        "r",
        "á»¥"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "##h",
        "##ụ",
        "##r",
        "##ụ"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọhụrụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 75,
      "category": "adjectives",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "new",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁new"
      ],
      "english_equivalent": "new",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "O",
        "lu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "O",
        "lu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "O",
        "lu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Olu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ol",
        "uw",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ol",
        "##uwa",
        "##se",
        "##un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwaseun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Olu",
        "wa",
        "se",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Ol",
        "ú",
        "w",
        "á",
        "�",
        "�",
        "�",
        "e",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.8
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 6,
      "token_splits": [
        "Ol",
        "ú",
        "w",
        "á",
        "ṣe",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "Ol",
        "Ãº",
        "w",
        "Ã¡",
        "á¹",
        "£",
        "e",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Ol",
        "ú",
        "w",
        "á",
        "ṣ",
        "eun"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁Ol",
        "ú",
        "w",
        "á",
        "ṣ",
        "e",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Ol",
        "##ú",
        "##w",
        "##á",
        "##ṣ",
        "##eun"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáṣeun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 7,
      "token_splits": [
        "▁O",
        "lú",
        "w",
        "á",
        "ṣ",
        "e",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Must",
        "aph",
        "a"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.6
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Must",
        "apha"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.4
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Must",
        "aph",
        "a"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.6
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Must",
        "apha"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Must",
        "aph",
        "a"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Must",
        "##ap",
        "##ha"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.75
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Mustapha",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Musta",
        "pha"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "Ch",
        "uk",
        "wu",
        "em",
        "eka"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ch",
        "ukwu",
        "em",
        "eka"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.8
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "Ch",
        "uk",
        "wu",
        "em",
        "eka"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Ch",
        "uk",
        "w",
        "ue",
        "me",
        "ka"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁Ch",
        "uk",
        "w",
        "u",
        "em",
        "ek",
        "a"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "Chu",
        "##k",
        "##wu",
        "##eme",
        "##ka"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Chukwuemeka",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Chu",
        "kw",
        "u",
        "eme",
        "ka"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 5,
      "token_splits": [
        "O",
        "lu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "O",
        "lu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 5,
      "token_splits": [
        "O",
        "lu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Olu",
        "w",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ol",
        "uw",
        "ase",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ol",
        "##uwa",
        "##se",
        "##un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 76,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Oluwaseun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁Olu",
        "wa",
        "se",
        "un"
      ],
      "english_equivalent": "Oluwaseun",
      "english_tokens": 4,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "T",
        "emit",
        "ope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Tem",
        "ito",
        "pe"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "T",
        "emit",
        "ope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Tem",
        "itope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Tem",
        "it",
        "ope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Tem",
        "##ito",
        "##pe"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Temitope",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Tem",
        "ito",
        "pe"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "T",
        "�",
        "�",
        "́",
        "m",
        "ít",
        "ọ",
        "́",
        "p",
        "�",
        "�",
        "́"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "T",
        "ẹ",
        "́",
        "m",
        "ít",
        "ọ",
        "́",
        "p",
        "ẹ",
        "́"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "T",
        "áº¹",
        "Ìģ",
        "mÃŃt",
        "á»į",
        "Ìģ",
        "p",
        "áº¹",
        "Ìģ"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "T",
        "ẹ",
        "́",
        "m",
        "ít",
        "ọ",
        "́",
        "p",
        "ẹ",
        "́"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁T",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "m",
        "ít",
        "ọ",
        "́",
        "p",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 4.667
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "T",
        "##ẹ́",
        "##mí",
        "##t",
        "##ọ́",
        "##p",
        "##ẹ́"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Tẹ́mítọ́pẹ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁T",
        "ẹ",
        "́",
        "mít",
        "ọ",
        "́",
        "p",
        "ẹ",
        "́"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "A",
        "min",
        "u"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "A",
        "min",
        "u"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "A",
        "min",
        "u"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "A",
        "minu"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁A",
        "min",
        "u"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Amin",
        "##u"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Aminu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Amin",
        "u"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "�",
        "�",
        "b",
        "ị",
        "n",
        "na"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ọ",
        "b",
        "ịn",
        "na"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "á»",
        "Į",
        "b",
        "á»ĭ",
        "n",
        "na"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "Ọ",
        "b",
        "ịn",
        "na"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 8,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "b",
        "ị",
        "n",
        "na"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "Ọ",
        "##b",
        "##ị",
        "##nna"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọbịnna",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁",
        "Ọ",
        "b",
        "ị",
        "nna"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "T",
        "emit",
        "ope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Tem",
        "ito",
        "pe"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "T",
        "emit",
        "ope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Tem",
        "itope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Tem",
        "it",
        "ope"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Tem",
        "##ito",
        "##pe"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 77,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Temitope",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Tem",
        "ito",
        "pe"
      ],
      "english_equivalent": "Temitope",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "O",
        "lu",
        "w",
        "af",
        "un",
        "mil",
        "ayo"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 7,
      "token_splits": [
        "O",
        "lu",
        "w",
        "af",
        "un",
        "mil",
        "ayo"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "O",
        "lu",
        "w",
        "af",
        "un",
        "mil",
        "ayo"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "Olu",
        "waf",
        "un",
        "mi",
        "lay",
        "o"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁Ol",
        "u",
        "wa",
        "fun",
        "mil",
        "ay",
        "o"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "Ol",
        "##uwa",
        "##fu",
        "##n",
        "##mila",
        "##yo"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "eng",
      "language_name": "English",
      "text": "Oluwafunmilayo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁Olu",
        "wa",
        "fun",
        "mi",
        "layo"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 5,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Ol",
        "ú",
        "w",
        "áf",
        "ú",
        "nm",
        "il",
        "á",
        "y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Ol",
        "ú",
        "w",
        "á",
        "f",
        "ún",
        "mil",
        "á",
        "y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "Ol",
        "Ãº",
        "w",
        "Ã¡f",
        "Ãº",
        "nm",
        "il",
        "Ã¡y",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "Ol",
        "ú",
        "w",
        "á",
        "f",
        "ún",
        "mil",
        "áy",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Ol",
        "ú",
        "w",
        "á",
        "f",
        "ún",
        "mil",
        "á",
        "y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 1.571
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "Ol",
        "##ú",
        "##w",
        "##á",
        "##f",
        "##ún",
        "##mil",
        "##áy",
        "##ọ̀"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Olúwáfúnmiláyọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁O",
        "lú",
        "w",
        "á",
        "f",
        "ún",
        "mi",
        "lá",
        "y",
        "ọ",
        "̀"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 5,
      "token_ratio_vs_english": 2.2
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Z",
        "ain",
        "ab"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Z",
        "ain",
        "ab"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Z",
        "ain",
        "ab"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Z",
        "ain",
        "ab"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Z",
        "ain",
        "ab"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Za",
        "##ina",
        "##b"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Zainab",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Za",
        "in",
        "ab"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.6
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ad",
        "ae",
        "ze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ada",
        "e",
        "ze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Ad",
        "ae",
        "ze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "A",
        "dae",
        "ze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁A",
        "da",
        "eze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.429
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Ada",
        "##eze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Adaeze",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Ada",
        "eze"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.4
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Fun",
        "mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.286
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Fun",
        "mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.286
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Fun",
        "mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.286
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Fun",
        "mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Fun",
        "mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 7,
      "token_ratio_vs_english": 0.286
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Fun",
        "##mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 6,
      "token_ratio_vs_english": 0.333
    },
    {
      "entry_id": 78,
      "category": "names",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Funmi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Fun",
        "mi"
      ],
      "english_equivalent": "Oluwafunmilayo",
      "english_tokens": 5,
      "token_ratio_vs_english": 0.4
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        " Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        " Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        "ĠHar",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Port",
        "▁Harcourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Port",
        "▁Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Port",
        "Harcourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Port Harcourt",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Port",
        "▁Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        " Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        " Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        "ĠHar",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 2,
      "token_splits": [
        "Port",
        "▁Harcourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁Port",
        "▁Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Port",
        "Harcourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Port Harcourt",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Port",
        "▁Har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "S",
        "ok",
        "oto"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "S",
        "ok",
        "oto"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "S",
        "ok",
        "oto"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "S",
        "oko",
        "to"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁S",
        "ok",
        "oto"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "So",
        "##kot",
        "##o"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Sokoto",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Sok",
        "oto"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 0.667
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "�",
        "�",
        "wer",
        "ri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ọ",
        "w",
        "erri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "á»",
        "Į",
        "wer",
        "ri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Ọ",
        "wer",
        "ri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "wer",
        "ri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ọ",
        "##wer",
        "##ri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọwerri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁",
        "Ọ",
        "wer",
        "ri"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "P",
        "orth",
        "arc",
        "ourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "P",
        "orth",
        "arc",
        "ourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "P",
        "orth",
        "arc",
        "ourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "P",
        "orth",
        "arc",
        "ourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁P",
        "orth",
        "arc",
        "ourt"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Port",
        "##har",
        "##court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 79,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Portharcourt",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Port",
        "har",
        "court"
      ],
      "english_equivalent": "Port Harcourt",
      "english_tokens": 3,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Abuja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Abu",
        "##ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "eng",
      "language_name": "English",
      "text": "Abuja",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "ù",
        "j",
        "á"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "Ab",
        "ù",
        "já"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "Ab",
        "Ã¹",
        "j",
        "Ã¡"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "Ab",
        "ù",
        "já"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁Ab",
        "ù",
        "j",
        "á"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "Ab",
        "##ù",
        "##já"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Abùjá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁Ab",
        "ù",
        "já"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Abuja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Abu",
        "##ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abuja",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Abuja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Abu",
        "##ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Abuja",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "Ab",
        "uja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "Abuja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "Abu",
        "##ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 80,
      "category": "places",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abuja",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁Abu",
        "ja"
      ],
      "english_equivalent": "Abuja",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "A",
        " child",
        " who",
        " wash",
        "es",
        " their",
        " hands",
        " well",
        " eats",
        " with",
        " elders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "A",
        " child",
        " who",
        " washes",
        " their",
        " hands",
        " well",
        " eats",
        " with",
        " elders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "A",
        "Ġchild",
        "Ġwho",
        "Ġwash",
        "es",
        "Ġtheir",
        "Ġhands",
        "Ġwell",
        "Ġeats",
        "Ġwith",
        "Ġelders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "A",
        "▁child",
        "▁who",
        "▁washes",
        "▁their",
        "▁hands",
        "▁well",
        "▁eats",
        "▁with",
        "▁elders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁A",
        "▁child",
        "▁who",
        "▁was",
        "hes",
        "▁their",
        "▁hands",
        "▁well",
        "▁e",
        "ats",
        "▁with",
        "▁el",
        "ders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "A",
        "child",
        "who",
        "was",
        "##hes",
        "their",
        "hands",
        "well",
        "eat",
        "##s",
        "with",
        "elders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "A child who washes their hands well eats with elders",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁A",
        "▁child",
        "▁who",
        "▁was",
        "hes",
        "▁their",
        "▁hands",
        "▁well",
        "▁e",
        "ats",
        "▁with",
        "▁el",
        "ders"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 34,
      "token_splits": [
        "�",
        "�",
        "m",
        "ọ",
        " tí",
        " ó",
        " b",
        "á",
        " w",
        "�",
        "�",
        " ",
        "ọ",
        "w",
        "ọ",
        "́",
        " r",
        "�",
        "�",
        "̀",
        " m",
        "ọ",
        "́",
        ",",
        " á",
        " b",
        "á",
        " à",
        "gb",
        "à",
        " j",
        "�",
        "�",
        "un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.091
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 21,
      "token_splits": [
        "Ọ",
        "mọ",
        " tí",
        " ó",
        " bá",
        " w",
        "ẹ",
        " ọwọ",
        "́",
        " rẹ",
        "̀",
        " mọ",
        "́",
        ",",
        " á",
        " bá",
        " à",
        "gb",
        "à",
        " jẹ",
        "un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.1
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 31,
      "token_splits": [
        "á»",
        "Į",
        "m",
        "á»į",
        "ĠtÃŃ",
        "ĠÃ³",
        "Ġb",
        "Ã¡",
        "Ġw",
        "áº¹",
        "Ġ",
        "á»į",
        "w",
        "á»į",
        "Ìģ",
        "Ġr",
        "áº¹",
        "ÌĢ",
        "Ġm",
        "á»į",
        "Ìģ",
        ",",
        "ĠÃ¡",
        "Ġb",
        "Ã¡",
        "ĠÃł",
        "gb",
        "Ãł",
        "Ġj",
        "áº¹",
        "un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.818
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 27,
      "token_splits": [
        "Ọ",
        "m",
        "ọ",
        "▁tí",
        "▁ó",
        "▁bá",
        "▁w",
        "ẹ",
        "▁ọ",
        "w",
        "ọ",
        "́",
        "▁r",
        "ẹ",
        "̀",
        "▁m",
        "ọ",
        "́",
        ",",
        "▁á",
        "▁bá",
        "▁à",
        "gb",
        "à",
        "▁j",
        "ẹ",
        "un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.7
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 40,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "m",
        "ọ",
        "▁t",
        "í",
        "▁ó",
        "▁b",
        "á",
        "▁w",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "▁",
        "ọ",
        "w",
        "ọ",
        "́",
        "▁r",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "▁m",
        "ọ",
        "́",
        ",",
        "▁á",
        "▁b",
        "á",
        "▁à",
        "gb",
        "à",
        "▁j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 3.077
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 24,
      "token_splits": [
        "Ọ",
        "##m",
        "##ọ",
        "tí",
        "ó",
        "bá",
        "w",
        "##ẹ",
        "ọ",
        "##w",
        "##ọ́",
        "r",
        "##ẹ̀",
        "m",
        "##ọ́",
        ",",
        "á",
        "bá",
        "à",
        "##g",
        "##bà",
        "j",
        "##ẹ",
        "##un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọmọ tí ó bá wẹ ọwọ́ rẹ̀ mọ́, á bá àgbà jẹun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 29,
      "token_splits": [
        "▁",
        "Ọ",
        "m",
        "ọ",
        "▁tí",
        "▁ó",
        "▁bá",
        "▁w",
        "ẹ",
        "▁",
        "ọ",
        "w",
        "ọ",
        "́",
        "▁r",
        "ẹ",
        "̀",
        "▁m",
        "ọ",
        "́",
        ",",
        "▁á",
        "▁bá",
        "▁à",
        "gb",
        "à",
        "▁j",
        "ẹ",
        "un"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.231
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 17,
      "token_splits": [
        "Y",
        "aro",
        " da",
        " ya",
        " wan",
        "ke",
        " hann",
        "un",
        "sa",
        " z",
        "ai",
        " ci",
        " ab",
        "inci",
        " da",
        " many",
        "a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.545
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Y",
        "aro",
        " da",
        " ya",
        " w",
        "anke",
        " hann",
        "unsa",
        " zai",
        " ci",
        " ab",
        "inci",
        " da",
        " many",
        "a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 17,
      "token_splits": [
        "Y",
        "aro",
        "Ġda",
        "Ġya",
        "Ġwan",
        "ke",
        "Ġhann",
        "un",
        "sa",
        "Ġz",
        "ai",
        "Ġci",
        "Ġab",
        "inci",
        "Ġda",
        "Ġmany",
        "a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.545
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 17,
      "token_splits": [
        "Y",
        "aro",
        "▁da",
        "▁ya",
        "▁wan",
        "ke",
        "▁hann",
        "un",
        "sa",
        "▁z",
        "ai",
        "▁ci",
        "▁ab",
        "inci",
        "▁da",
        "▁many",
        "a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.7
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 20,
      "token_splits": [
        "▁Y",
        "aro",
        "▁da",
        "▁ya",
        "▁w",
        "an",
        "ke",
        "▁h",
        "ann",
        "un",
        "sa",
        "▁z",
        "ai",
        "▁ci",
        "▁ab",
        "in",
        "ci",
        "▁da",
        "▁many",
        "a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.538
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 17,
      "token_splits": [
        "Ya",
        "##ro",
        "da",
        "ya",
        "wa",
        "##nke",
        "hann",
        "##uns",
        "##a",
        "za",
        "##i",
        "ci",
        "ab",
        "##inci",
        "da",
        "many",
        "##a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.417
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Yaro da ya wanke hannunsa zai ci abinci da manya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Ya",
        "ro",
        "▁da",
        "▁ya",
        "▁wan",
        "ke",
        "▁hann",
        "un",
        "sa",
        "▁zai",
        "▁ci",
        "▁abin",
        "ci",
        "▁da",
        "▁many",
        "a"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.231
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 22,
      "token_splits": [
        "N",
        "wa",
        " b",
        "ụ",
        " on",
        "ye",
        " sach",
        "ara",
        " aka",
        " ya",
        ",",
        " ya",
        " na",
        " nd",
        "ị",
        " o",
        "ken",
        "ye",
        " r",
        "ie",
        " n",
        "ri"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "N",
        "wa",
        " bụ",
        " onye",
        " sach",
        "ara",
        " aka",
        " ya",
        ",",
        " ya",
        " na",
        " ndị",
        " o",
        "ken",
        "ye",
        " rie",
        " nri"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.7
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "wa",
        "Ġbá»¥",
        "Ġon",
        "ye",
        "Ġsach",
        "ara",
        "Ġaka",
        "Ġya",
        ",",
        "Ġya",
        "Ġna",
        "Ġnd",
        "á»ĭ",
        "Ġo",
        "ken",
        "ye",
        "Ġr",
        "ie",
        "Ġn",
        "ri"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.909
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 20,
      "token_splits": [
        "N",
        "wa",
        "▁bụ",
        "▁on",
        "ye",
        "▁sach",
        "ara",
        "▁aka",
        "▁ya",
        ",",
        "▁ya",
        "▁na",
        "▁nd",
        "ị",
        "▁o",
        "ken",
        "ye",
        "▁rie",
        "▁n",
        "ri"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 24,
      "token_splits": [
        "▁N",
        "wa",
        "▁b",
        "ụ",
        "▁on",
        "ye",
        "▁s",
        "ach",
        "ara",
        "▁a",
        "ka",
        "▁ya",
        ",",
        "▁ya",
        "▁na",
        "▁nd",
        "ị",
        "▁o",
        "ken",
        "ye",
        "▁ri",
        "e",
        "▁n",
        "ri"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.846
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 23,
      "token_splits": [
        "N",
        "##wa",
        "b",
        "##ụ",
        "on",
        "##ye",
        "sac",
        "##hara",
        "aka",
        "ya",
        ",",
        "ya",
        "na",
        "n",
        "##d",
        "##ị",
        "ok",
        "##eny",
        "##e",
        "ri",
        "##e",
        "nr",
        "##i"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.917
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nwa bụ onye sachara aka ya, ya na ndị okenye rie nri",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 22,
      "token_splits": [
        "▁N",
        "wa",
        "▁b",
        "ụ",
        "▁on",
        "ye",
        "▁sac",
        "hara",
        "▁aka",
        "▁ya",
        ",",
        "▁ya",
        "▁na",
        "▁",
        "nd",
        "ị",
        "▁o",
        "ken",
        "ye",
        "▁rie",
        "▁nr",
        "i"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.692
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "P",
        "ik",
        "in",
        " we",
        "y",
        " wash",
        " im",
        " hand",
        " well",
        " go",
        " chop",
        " with",
        " big",
        " people"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.273
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "P",
        "ikin",
        " we",
        "y",
        " wash",
        " im",
        " hand",
        " well",
        " go",
        " chop",
        " with",
        " big",
        " people"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.3
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "P",
        "ik",
        "in",
        "Ġwe",
        "y",
        "Ġwash",
        "Ġim",
        "Ġhand",
        "Ġwell",
        "Ġgo",
        "Ġchop",
        "Ġwith",
        "Ġbig",
        "Ġpeople"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.273
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "P",
        "ikin",
        "▁wey",
        "▁wash",
        "▁im",
        "▁hand",
        "▁well",
        "▁go",
        "▁chop",
        "▁with",
        "▁big",
        "▁people"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁P",
        "ik",
        "in",
        "▁we",
        "y",
        "▁wash",
        "▁im",
        "▁hand",
        "▁well",
        "▁go",
        "▁chop",
        "▁with",
        "▁big",
        "▁people"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.077
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "Pi",
        "##kin",
        "we",
        "##y",
        "was",
        "##h",
        "im",
        "hand",
        "well",
        "go",
        "cho",
        "##p",
        "with",
        "big",
        "people"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 81,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Pikin wey wash im hand well go chop with big people",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁Pi",
        "kin",
        "▁we",
        "y",
        "▁was",
        "h",
        "▁im",
        "▁hand",
        "▁well",
        "▁go",
        "▁",
        "chop",
        "▁with",
        "▁big",
        "▁people"
      ],
      "english_equivalent": "A child who washes their hands well eats with elders",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.154
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "However",
        " long",
        " the",
        " night",
        ",",
        " the",
        " dawn",
        " will",
        " break"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "However",
        " long",
        " the",
        " night",
        ",",
        " the",
        " dawn",
        " will",
        " break"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "However",
        "Ġlong",
        "Ġthe",
        "Ġnight",
        ",",
        "Ġthe",
        "Ġdawn",
        "Ġwill",
        "Ġbreak"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "However",
        "▁long",
        "▁the",
        "▁night",
        ",",
        "▁the",
        "▁dawn",
        "▁will",
        "▁break"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁However",
        "▁long",
        "▁the",
        "▁night",
        ",",
        "▁the",
        "▁dawn",
        "▁will",
        "▁break"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "However",
        "long",
        "the",
        "night",
        ",",
        "the",
        "da",
        "##wn",
        "will",
        "break"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "However long the night, the dawn will break",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁However",
        "▁long",
        "▁the",
        "▁night",
        ",",
        "▁the",
        "▁daw",
        "n",
        "▁will",
        "▁break"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 24,
      "token_splits": [
        "B",
        "í",
        " al",
        "�",
        "�",
        "́",
        " b",
        "á",
        " p",
        "�",
        "�",
        "́",
        " t",
        "ó",
        ",",
        " ",
        "ọ",
        "̀",
        "s",
        "án",
        " á",
        " m",
        "ọ",
        "́"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 18,
      "token_splits": [
        "B",
        "í",
        " al",
        "ẹ",
        "́",
        " bá",
        " p",
        "ẹ",
        "́",
        " tó",
        ",",
        " ọ",
        "̀",
        "s",
        "án",
        " á",
        " mọ",
        "́"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 22,
      "token_splits": [
        "B",
        "ÃŃ",
        "Ġal",
        "áº¹",
        "Ìģ",
        "Ġb",
        "Ã¡",
        "Ġp",
        "áº¹",
        "Ìģ",
        "Ġt",
        "Ã³",
        ",",
        "Ġ",
        "á»į",
        "ÌĢ",
        "s",
        "Ã¡n",
        "ĠÃ¡",
        "Ġm",
        "á»į",
        "Ìģ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.444
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 19,
      "token_splits": [
        "B",
        "í",
        "▁al",
        "ẹ",
        "́",
        "▁bá",
        "▁p",
        "ẹ",
        "́",
        "▁tó",
        ",",
        "▁ọ",
        "̀",
        "s",
        "án",
        "▁á",
        "▁m",
        "ọ",
        "́"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.111
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁B",
        "í",
        "▁al",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁b",
        "á",
        "▁p",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁t",
        "ó",
        ",",
        "▁",
        "ọ",
        "̀",
        "s",
        "án",
        "▁á",
        "▁m",
        "ọ",
        "́"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.889
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "Bí",
        "al",
        "##ẹ́",
        "bá",
        "p",
        "##ẹ́",
        "tó",
        ",",
        "ọ",
        "##̀",
        "##s",
        "##án",
        "á",
        "m",
        "##ọ́"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Bí alẹ́ bá pẹ́ tó, ọ̀sán á mọ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 19,
      "token_splits": [
        "▁Bí",
        "▁al",
        "ẹ",
        "́",
        "▁bá",
        "▁p",
        "ẹ",
        "́",
        "▁tó",
        ",",
        "▁",
        "ọ",
        "̀",
        "s",
        "án",
        "▁á",
        "▁m",
        "ọ",
        "́"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.9
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "K",
        "o",
        " dare",
        " ya",
        " yi",
        " ts",
        "aw",
        "o",
        ",",
        " g",
        "ari",
        " z",
        "ai",
        " way",
        "e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Ko",
        " dare",
        " ya",
        " yi",
        " ts",
        "awo",
        ",",
        " gari",
        " zai",
        " way",
        "e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "K",
        "o",
        "Ġdare",
        "Ġya",
        "Ġyi",
        "Ġts",
        "aw",
        "o",
        ",",
        "Ġg",
        "ari",
        "Ġz",
        "ai",
        "Ġway",
        "e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "Ko",
        "▁dare",
        "▁ya",
        "▁yi",
        "▁tsa",
        "wo",
        ",",
        "▁gari",
        "▁z",
        "ai",
        "▁way",
        "e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁Ko",
        "▁dare",
        "▁ya",
        "▁y",
        "i",
        "▁t",
        "sa",
        "wo",
        ",",
        "▁g",
        "ari",
        "▁z",
        "ai",
        "▁way",
        "e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "Ko",
        "dare",
        "ya",
        "yi",
        "t",
        "##saw",
        "##o",
        ",",
        "gar",
        "##i",
        "za",
        "##i",
        "way",
        "##e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Ko dare ya yi tsawo, gari zai waye",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Ko",
        "▁dare",
        "▁ya",
        "▁yi",
        "▁tsa",
        "wo",
        ",",
        "▁gari",
        "▁zai",
        "▁way",
        "e"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "O",
        "tu",
        " ab",
        "al",
        "ị",
        " si",
        " og",
        "olog",
        "o",
        ",",
        " chi",
        " ga",
        "-ab",
        "ọ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "O",
        "tu",
        " abal",
        "ị",
        " si",
        " ogologo",
        ",",
        " chi",
        " ga",
        "-ab",
        "ọ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "O",
        "tu",
        "Ġab",
        "al",
        "á»ĭ",
        "Ġsi",
        "Ġog",
        "olog",
        "o",
        ",",
        "Ġchi",
        "Ġga",
        "-ab",
        "á»į"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 15,
      "token_splits": [
        "O",
        "tu",
        "▁ab",
        "al",
        "ị",
        "▁si",
        "▁o",
        "go",
        "logo",
        ",",
        "▁chi",
        "▁ga",
        "-",
        "ab",
        "ọ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁Ot",
        "u",
        "▁ab",
        "al",
        "ị",
        "▁si",
        "▁og",
        "olog",
        "o",
        ",",
        "▁chi",
        "▁ga",
        "-",
        "ab",
        "ọ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "O",
        "##tu",
        "ab",
        "##al",
        "##ị",
        "si",
        "og",
        "##ologo",
        ",",
        "chi",
        "ga",
        "-",
        "ab",
        "##ọ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Otu abalị si ogologo, chi ga-abọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁O",
        "tu",
        "▁a",
        "bal",
        "ị",
        "▁si",
        "▁og",
        "olog",
        "o",
        ",",
        "▁chi",
        "▁ga",
        "-",
        "ab",
        "ọ"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "No",
        " matter",
        " how",
        " long",
        " night",
        " be",
        ",",
        " morning",
        " go",
        " still",
        " come"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "No",
        " matter",
        " how",
        " long",
        " night",
        " be",
        ",",
        " morning",
        " go",
        " still",
        " come"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "No",
        "Ġmatter",
        "Ġhow",
        "Ġlong",
        "Ġnight",
        "Ġbe",
        ",",
        "Ġmorning",
        "Ġgo",
        "Ġstill",
        "Ġcome"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "No",
        "▁matter",
        "▁how",
        "▁long",
        "▁night",
        "▁be",
        ",",
        "▁morning",
        "▁go",
        "▁still",
        "▁come"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁No",
        "▁matter",
        "▁how",
        "▁long",
        "▁night",
        "▁be",
        ",",
        "▁morning",
        "▁go",
        "▁still",
        "▁come"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "No",
        "matter",
        "how",
        "long",
        "night",
        "be",
        ",",
        "morning",
        "go",
        "still",
        "come"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 82,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "No matter how long night be, morning go still come",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁No",
        "▁matter",
        "▁how",
        "▁long",
        "▁night",
        "▁be",
        ",",
        "▁morning",
        "▁go",
        "▁still",
        "▁come"
      ],
      "english_equivalent": "However long the night, the dawn will break",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "What",
        " an",
        " elder",
        " sees",
        " sitting",
        " down",
        ",",
        " a",
        " child",
        " cannot",
        " see",
        " standing",
        " up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "What",
        " an",
        " elder",
        " sees",
        " sitting",
        " down",
        ",",
        " a",
        " child",
        " cannot",
        " see",
        " standing",
        " up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "What",
        "Ġan",
        "Ġelder",
        "Ġsees",
        "Ġsitting",
        "Ġdown",
        ",",
        "Ġa",
        "Ġchild",
        "Ġcannot",
        "Ġsee",
        "Ġstanding",
        "Ġup"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "What",
        "▁an",
        "▁elder",
        "▁sees",
        "▁sitting",
        "▁down",
        ",",
        "▁a",
        "▁child",
        "▁cannot",
        "▁see",
        "▁standing",
        "▁up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁What",
        "▁an",
        "▁elder",
        "▁sees",
        "▁sitting",
        "▁down",
        ",",
        "▁a",
        "▁child",
        "▁cannot",
        "▁see",
        "▁standing",
        "▁up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "What",
        "an",
        "elder",
        "sees",
        "sitting",
        "down",
        ",",
        "a",
        "child",
        "cannot",
        "see",
        "standing",
        "up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "eng",
      "language_name": "English",
      "text": "What an elder sees sitting down, a child cannot see standing up",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁What",
        "▁an",
        "▁el",
        "der",
        "▁se",
        "es",
        "▁sitting",
        "▁down",
        ",",
        "▁a",
        "▁child",
        "▁cannot",
        "▁see",
        "▁standing",
        "▁up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 35,
      "token_splits": [
        "Oh",
        "un",
        " tí",
        " à",
        "gb",
        "à",
        " b",
        "á",
        " r",
        "í",
        " jó",
        "k",
        "ò",
        "ó",
        ",",
        " tí",
        " ",
        "ọ",
        "m",
        "ọ",
        "d",
        "é",
        " b",
        "á",
        " d",
        "ú",
        "ró",
        ",",
        " k",
        "ò",
        " l",
        "è",
        " r",
        "í",
        " i"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.692
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 26,
      "token_splits": [
        "Oh",
        "un",
        " tí",
        " à",
        "gb",
        "à",
        " bá",
        " rí",
        " jó",
        "k",
        "ò",
        "ó",
        ",",
        " tí",
        " ọ",
        "m",
        "ọd",
        "é",
        " bá",
        " dú",
        "ró",
        ",",
        " kò",
        " lè",
        " rí",
        " i"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 35,
      "token_splits": [
        "Oh",
        "un",
        "ĠtÃŃ",
        "ĠÃł",
        "gb",
        "Ãł",
        "Ġb",
        "Ã¡",
        "Ġr",
        "ÃŃ",
        "ĠjÃ³",
        "k",
        "Ã²",
        "Ã³",
        ",",
        "ĠtÃŃ",
        "Ġ",
        "á»į",
        "m",
        "á»į",
        "d",
        "Ã©",
        "Ġb",
        "Ã¡",
        "Ġd",
        "Ãº",
        "rÃ³",
        ",",
        "Ġk",
        "Ã²",
        "Ġl",
        "Ã¨",
        "Ġr",
        "ÃŃ",
        "Ġi"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.692
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 28,
      "token_splits": [
        "Oh",
        "un",
        "▁tí",
        "▁à",
        "gb",
        "à",
        "▁bá",
        "▁rí",
        "▁jó",
        "k",
        "ò",
        "ó",
        ",",
        "▁tí",
        "▁ọ",
        "m",
        "ọ",
        "dé",
        "▁bá",
        "▁dú",
        "ró",
        ",",
        "▁k",
        "ò",
        "▁l",
        "è",
        "▁rí",
        "▁i"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.154
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 38,
      "token_splits": [
        "▁Oh",
        "un",
        "▁t",
        "í",
        "▁à",
        "gb",
        "à",
        "▁b",
        "á",
        "▁r",
        "í",
        "▁j",
        "ó",
        "k",
        "ò",
        "ó",
        ",",
        "▁t",
        "í",
        "▁",
        "ọ",
        "m",
        "ọ",
        "d",
        "é",
        "▁b",
        "á",
        "▁d",
        "ú",
        "ró",
        ",",
        "▁k",
        "ò",
        "▁l",
        "è",
        "▁r",
        "í",
        "▁i"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.923
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 30,
      "token_splits": [
        "Oh",
        "##un",
        "tí",
        "à",
        "##g",
        "##bà",
        "bá",
        "r",
        "##í",
        "jó",
        "##k",
        "##ò",
        "##ó",
        ",",
        "tí",
        "ọ",
        "##m",
        "##ọ",
        "##dé",
        "bá",
        "d",
        "##úr",
        "##ó",
        ",",
        "kò",
        "l",
        "##è",
        "r",
        "##í",
        "i"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.308
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ohun tí àgbà bá rí jókòó, tí ọmọdé bá dúró, kò lè rí i",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 29,
      "token_splits": [
        "▁O",
        "hun",
        "▁tí",
        "▁à",
        "gb",
        "à",
        "▁bá",
        "▁rí",
        "▁jó",
        "k",
        "ò",
        "ó",
        ",",
        "▁tí",
        "▁",
        "ọ",
        "m",
        "ọ",
        "dé",
        "▁bá",
        "▁dú",
        "ró",
        ",",
        "▁k",
        "ò",
        "▁",
        "lè",
        "▁rí",
        "▁i"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.933
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 26,
      "token_splits": [
        "Ab",
        "in",
        " da",
        " bab",
        "ba",
        " ya",
        " g",
        "ani",
        " a",
        " za",
        "une",
        ",",
        " y",
        "aro",
        " ba",
        " z",
        "ai",
        " i",
        "ya",
        " gan",
        "in",
        " sa",
        " a",
        " ts",
        "aye",
        " ba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 22,
      "token_splits": [
        "Ab",
        "in",
        " da",
        " bab",
        "ba",
        " ya",
        " gani",
        " a",
        " za",
        "une",
        ",",
        " y",
        "aro",
        " ba",
        " zai",
        " iya",
        " ganin",
        " sa",
        " a",
        " ts",
        "aye",
        " ba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.692
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 26,
      "token_splits": [
        "Ab",
        "in",
        "Ġda",
        "Ġbab",
        "ba",
        "Ġya",
        "Ġg",
        "ani",
        "Ġa",
        "Ġza",
        "une",
        ",",
        "Ġy",
        "aro",
        "Ġba",
        "Ġz",
        "ai",
        "Ġi",
        "ya",
        "Ġgan",
        "in",
        "Ġsa",
        "Ġa",
        "Ġts",
        "aye",
        "Ġba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 24,
      "token_splits": [
        "Ab",
        "in",
        "▁da",
        "▁bab",
        "ba",
        "▁ya",
        "▁gani",
        "▁a",
        "▁zau",
        "ne",
        ",",
        "▁y",
        "aro",
        "▁ba",
        "▁z",
        "ai",
        "▁iya",
        "▁gan",
        "in",
        "▁sa",
        "▁a",
        "▁ts",
        "aye",
        "▁ba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.846
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁Ab",
        "in",
        "▁da",
        "▁bab",
        "ba",
        "▁ya",
        "▁g",
        "ani",
        "▁a",
        "▁za",
        "une",
        ",",
        "▁y",
        "aro",
        "▁ba",
        "▁z",
        "ai",
        "▁i",
        "ya",
        "▁gan",
        "in",
        "▁sa",
        "▁a",
        "▁ts",
        "aye",
        "▁ba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 27,
      "token_splits": [
        "Abi",
        "##n",
        "da",
        "ba",
        "##bba",
        "ya",
        "gan",
        "##i",
        "a",
        "za",
        "##une",
        ",",
        "ya",
        "##ro",
        "ba",
        "za",
        "##i",
        "i",
        "##ya",
        "gan",
        "##in",
        "sa",
        "a",
        "t",
        "##say",
        "##e",
        "ba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.077
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Abin da babba ya gani a zaune, yaro ba zai iya ganin sa a tsaye ba",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 23,
      "token_splits": [
        "▁Ab",
        "in",
        "▁da",
        "▁bab",
        "ba",
        "▁ya",
        "▁gani",
        "▁a",
        "▁za",
        "une",
        ",",
        "▁ya",
        "ro",
        "▁ba",
        "▁zai",
        "▁iya",
        "▁gan",
        "in",
        "▁sa",
        "▁a",
        "▁tsa",
        "ye",
        "▁ba"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.533
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 32,
      "token_splits": [
        "I",
        "he",
        " ag",
        "adi",
        " nw",
        "any",
        "ị",
        " n",
        "ọ",
        " al",
        "a",
        " na",
        "-",
        "ah",
        "ụ",
        ",",
        " nw",
        "ata",
        " ar",
        "ị",
        "a",
        " el",
        "u",
        " os",
        "isi",
        " ad",
        "ị",
        "gh",
        "ị",
        " ah",
        "ụ",
        " ya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.462
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 19,
      "token_splits": [
        "Ihe",
        " ag",
        "adi",
        " nwanyị",
        " nọ",
        " ala",
        " na",
        "-ahụ",
        ",",
        " nw",
        "ata",
        " ar",
        "ịa",
        " elu",
        " osisi",
        " ad",
        "ịghị",
        " ahụ",
        " ya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.462
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 31,
      "token_splits": [
        "I",
        "he",
        "Ġag",
        "adi",
        "Ġnw",
        "any",
        "á»ĭ",
        "Ġn",
        "á»į",
        "Ġal",
        "a",
        "Ġna",
        "-",
        "ah",
        "á»¥",
        ",",
        "Ġnw",
        "ata",
        "Ġar",
        "á»ĭa",
        "Ġel",
        "u",
        "Ġos",
        "isi",
        "Ġad",
        "á»ĭ",
        "gh",
        "á»ĭ",
        "Ġah",
        "á»¥",
        "Ġya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.385
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 29,
      "token_splits": [
        "I",
        "he",
        "▁ag",
        "adi",
        "▁n",
        "wany",
        "ị",
        "▁n",
        "ọ",
        "▁ala",
        "▁na",
        "-",
        "ah",
        "ụ",
        ",",
        "▁nw",
        "ata",
        "▁ar",
        "ịa",
        "▁elu",
        "▁os",
        "isi",
        "▁ad",
        "ị",
        "gh",
        "ị",
        "▁ah",
        "ụ",
        "▁ya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.231
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 34,
      "token_splits": [
        "▁I",
        "he",
        "▁ag",
        "adi",
        "▁n",
        "w",
        "any",
        "ị",
        "▁n",
        "ọ",
        "▁al",
        "a",
        "▁na",
        "-",
        "ah",
        "ụ",
        ",",
        "▁n",
        "w",
        "ata",
        "▁ar",
        "ị",
        "a",
        "▁el",
        "u",
        "▁os",
        "isi",
        "▁ad",
        "ị",
        "gh",
        "ị",
        "▁ah",
        "ụ",
        "▁ya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.615
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 31,
      "token_splits": [
        "I",
        "##he",
        "aga",
        "##di",
        "n",
        "##wany",
        "##ị",
        "n",
        "##ọ",
        "ala",
        "na",
        "-",
        "ah",
        "##ụ",
        ",",
        "n",
        "##wat",
        "##a",
        "ar",
        "##ị",
        "##a",
        "elu",
        "osi",
        "##si",
        "ad",
        "##ị",
        "##gh",
        "##ị",
        "ah",
        "##ụ",
        "ya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.385
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ihe agadi nwanyị nọ ala na-ahụ, nwata arịa elu osisi adịghị ahụ ya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 31,
      "token_splits": [
        "▁I",
        "he",
        "▁agad",
        "i",
        "▁n",
        "wany",
        "ị",
        "▁n",
        "ọ",
        "▁ala",
        "▁na",
        "-",
        "ah",
        "ụ",
        ",",
        "▁n",
        "wa",
        "ta",
        "▁ar",
        "ị",
        "a",
        "▁elu",
        "▁os",
        "isi",
        "▁ad",
        "ị",
        "gh",
        "ị",
        "▁ah",
        "ụ",
        "▁ya"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.067
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "W",
        "et",
        "in",
        " old",
        " person",
        " see",
        " sit",
        " down",
        ",",
        " small",
        " pik",
        "in",
        " no",
        " fit",
        " see",
        " am",
        " even",
        " if",
        " e",
        " stand",
        " up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.615
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 20,
      "token_splits": [
        "W",
        "etin",
        " old",
        " person",
        " see",
        " sit",
        " down",
        ",",
        " small",
        " p",
        "ikin",
        " no",
        " fit",
        " see",
        " am",
        " even",
        " if",
        " e",
        " stand",
        " up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.538
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 20,
      "token_splits": [
        "W",
        "etin",
        "Ġold",
        "Ġperson",
        "Ġsee",
        "Ġsit",
        "Ġdown",
        ",",
        "Ġsmall",
        "Ġpik",
        "in",
        "Ġno",
        "Ġfit",
        "Ġsee",
        "Ġam",
        "Ġeven",
        "Ġif",
        "Ġe",
        "Ġstand",
        "Ġup"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.538
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 20,
      "token_splits": [
        "Wet",
        "in",
        "▁old",
        "▁person",
        "▁see",
        "▁sit",
        "▁down",
        ",",
        "▁small",
        "▁pik",
        "in",
        "▁no",
        "▁fit",
        "▁see",
        "▁am",
        "▁even",
        "▁if",
        "▁e",
        "▁stand",
        "▁up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.538
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁W",
        "et",
        "in",
        "▁old",
        "▁person",
        "▁see",
        "▁sit",
        "▁down",
        ",",
        "▁small",
        "▁p",
        "ik",
        "in",
        "▁no",
        "▁fit",
        "▁see",
        "▁am",
        "▁even",
        "▁if",
        "▁e",
        "▁stand",
        "▁up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.692
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 20,
      "token_splits": [
        "Wet",
        "##in",
        "old",
        "person",
        "see",
        "sit",
        "down",
        ",",
        "small",
        "pi",
        "##kin",
        "no",
        "fit",
        "see",
        "am",
        "even",
        "if",
        "e",
        "stand",
        "up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.538
    },
    {
      "entry_id": 83,
      "category": "proverbs",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Wetin old person see sit down, small pikin no fit see am even if e stand up",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 20,
      "token_splits": [
        "▁We",
        "tin",
        "▁old",
        "▁person",
        "▁see",
        "▁sit",
        "▁down",
        ",",
        "▁small",
        "▁pik",
        "in",
        "▁no",
        "▁fit",
        "▁see",
        "▁am",
        "▁even",
        "▁if",
        "▁e",
        "▁stand",
        "▁up"
      ],
      "english_equivalent": "What an elder sees sitting down, a child cannot see standing up",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Please",
        " transfer",
        " five",
        " thousand",
        " n",
        "aira",
        " to",
        " my",
        " account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Please",
        " transfer",
        " five",
        " thousand",
        " na",
        "ira",
        " to",
        " my",
        " account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "Please",
        "Ġtransfer",
        "Ġfive",
        "Ġthousand",
        "Ġn",
        "aira",
        "Ġto",
        "Ġmy",
        "Ġaccount"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "Please",
        "▁transfer",
        "▁five",
        "▁thousand",
        "▁naira",
        "▁to",
        "▁my",
        "▁account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁Please",
        "▁transfer",
        "▁five",
        "▁thousand",
        "▁n",
        "air",
        "a",
        "▁to",
        "▁my",
        "▁account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "Please",
        "transfer",
        "five",
        "thousand",
        "nai",
        "##ra",
        "to",
        "my",
        "account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "Please transfer five thousand naira to my account",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁Please",
        "▁transfer",
        "▁five",
        "▁thousand",
        "▁na",
        "ira",
        "▁to",
        "▁my",
        "▁account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 47,
      "token_splits": [
        "�",
        "�",
        " j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        " fi",
        " ",
        "�",
        "�",
        "gb",
        "�",
        "�",
        "̀",
        "r",
        "ún",
        " már",
        "ù",
        "n",
        "-",
        "ún",
        " ow",
        "ó",
        " n",
        "á",
        "í",
        "r",
        "à",
        " r",
        "án",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́",
        " sí",
        " à",
        "k",
        "á",
        "�",
        "�",
        "t",
        "ì",
        " mi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 5.222
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 32,
      "token_splits": [
        "Ẹ",
        " j",
        "ọ",
        "̀",
        "wọ",
        "́",
        " fi",
        " ẹgbẹ",
        "̀",
        "r",
        "ún",
        " már",
        "ùn",
        "-",
        "ún",
        " ow",
        "ó",
        " ná",
        "ír",
        "à",
        " r",
        "án",
        "ṣẹ",
        "́",
        " sí",
        " à",
        "ká",
        "�",
        "�",
        "t",
        "ì",
        " mi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.556
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 40,
      "token_splits": [
        "áº",
        "¸",
        "Ġj",
        "á»į",
        "ÌĢ",
        "w",
        "á»į",
        "Ìģ",
        "Ġfi",
        "Ġ",
        "áº¹",
        "gb",
        "áº¹",
        "ÌĢ",
        "r",
        "Ãºn",
        "ĠmÃ¡r",
        "Ã¹",
        "n",
        "-",
        "Ãºn",
        "Ġow",
        "Ã³",
        "ĠnÃ¡",
        "ÃŃr",
        "Ãł",
        "Ġr",
        "Ã¡n",
        "á¹",
        "£",
        "áº¹",
        "Ìģ",
        "ĠsÃŃ",
        "ĠÃł",
        "kÃ¡",
        "Ç",
        "¹",
        "t",
        "Ã¬",
        "Ġmi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 4.444
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 36,
      "token_splits": [
        "Ẹ",
        "▁j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        "▁fi",
        "▁",
        "ẹ",
        "gb",
        "ẹ",
        "̀",
        "r",
        "ún",
        "▁már",
        "ùn",
        "-",
        "ún",
        "▁ow",
        "ó",
        "▁ná",
        "ír",
        "à",
        "▁r",
        "án",
        "ṣ",
        "ẹ",
        "́",
        "▁sí",
        "▁à",
        "ká",
        "ǹ",
        "t",
        "ì",
        "▁mi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 8,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 50,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "▁j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        "▁fi",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "gb",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ún",
        "▁m",
        "ár",
        "ù",
        "n",
        "-",
        "ún",
        "▁o",
        "w",
        "ó",
        "▁ná",
        "ír",
        "à",
        "▁r",
        "án",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁sí",
        "▁à",
        "k",
        "á",
        "<0xC7>",
        "<0xB9>",
        "t",
        "ì",
        "▁mi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 10,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 33,
      "token_splits": [
        "Ẹ",
        "j",
        "##ọ̀",
        "##w",
        "##ọ́",
        "fi",
        "ẹ",
        "##g",
        "##b",
        "##ẹ̀",
        "##rú",
        "##n",
        "már",
        "##ùn",
        "-",
        "ún",
        "o",
        "##w",
        "##ó",
        "ná",
        "##ír",
        "##à",
        "rá",
        "##n",
        "##ṣ",
        "##ẹ́",
        "sí",
        "à",
        "##ká",
        "##ǹ",
        "##t",
        "##ì",
        "mi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.667
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ jọ̀wọ́ fi ẹgbẹ̀rún márùn-ún owó náírà ránṣẹ́ sí àkáǹtì mi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 38,
      "token_splits": [
        "▁",
        "Ẹ",
        "▁j",
        "ọ",
        "̀",
        "w",
        "ọ",
        "́",
        "▁fi",
        "▁",
        "ẹ",
        "gb",
        "ẹ",
        "̀",
        "rún",
        "▁már",
        "ù",
        "n",
        "-",
        "ún",
        "▁",
        "ow",
        "ó",
        "▁ná",
        "í",
        "rà",
        "▁",
        "rán",
        "ṣ",
        "ẹ",
        "́",
        "▁sí",
        "▁à",
        "ká",
        "<unk>",
        "t",
        "ì",
        "▁mi"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 4.222
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Don",
        " Allah",
        " can",
        "ja",
        " n",
        "aira",
        " dub",
        "u",
        " bi",
        "yar",
        " zu",
        "wa",
        " as",
        "us",
        "una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 14,
      "token_splits": [
        "Don",
        " Allah",
        " can",
        "ja",
        " na",
        "ira",
        " dub",
        "u",
        " b",
        "iyar",
        " zuwa",
        " as",
        "us",
        "una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "Don",
        "ĠAllah",
        "Ġcan",
        "ja",
        "Ġn",
        "aira",
        "Ġdub",
        "u",
        "Ġbi",
        "yar",
        "Ġzu",
        "wa",
        "Ġas",
        "us",
        "una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "Don",
        "▁Allah",
        "▁can",
        "ja",
        "▁naira",
        "▁dub",
        "u",
        "▁bi",
        "yar",
        "▁zu",
        "wa",
        "▁asus",
        "una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.625
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 17,
      "token_splits": [
        "▁Don",
        "▁All",
        "ah",
        "▁can",
        "ja",
        "▁n",
        "air",
        "a",
        "▁dub",
        "u",
        "▁bi",
        "yar",
        "▁zu",
        "wa",
        "▁as",
        "us",
        "una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.7
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "Don",
        "Allah",
        "can",
        "##ja",
        "nai",
        "##ra",
        "dub",
        "##u",
        "bi",
        "##yar",
        "zu",
        "##wa",
        "asus",
        "##una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Don Allah canja naira dubu biyar zuwa asusuna",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁Don",
        "▁Allah",
        "▁can",
        "ja",
        "▁na",
        "ira",
        "▁du",
        "bu",
        "▁bi",
        "yar",
        "▁zuwa",
        "▁as",
        "us",
        "una"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 16,
      "token_splits": [
        "B",
        "iko",
        " z",
        "iga",
        " p",
        "uku",
        " ise",
        " n",
        "aira",
        " n",
        "'",
        "aka",
        "ụ",
        "nt",
        "ụ",
        " m"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.778
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "B",
        "iko",
        " z",
        "iga",
        " p",
        "uku",
        " ise",
        " na",
        "ira",
        " n",
        "'aka",
        "ụ",
        "nt",
        "ụ",
        " m"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 16,
      "token_splits": [
        "B",
        "iko",
        "Ġz",
        "iga",
        "Ġp",
        "uku",
        "Ġise",
        "Ġn",
        "aira",
        "Ġn",
        "'",
        "aka",
        "á»¥",
        "nt",
        "á»¥",
        "Ġm"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.778
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 15,
      "token_splits": [
        "B",
        "iko",
        "▁z",
        "iga",
        "▁p",
        "uku",
        "▁ise",
        "▁naira",
        "▁n",
        "'",
        "aka",
        "ụ",
        "nt",
        "ụ",
        "▁m"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.875
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 19,
      "token_splits": [
        "▁B",
        "iko",
        "▁z",
        "iga",
        "▁p",
        "uk",
        "u",
        "▁is",
        "e",
        "▁n",
        "air",
        "a",
        "▁n",
        "'",
        "aka",
        "ụ",
        "nt",
        "ụ",
        "▁m"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.9
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 16,
      "token_splits": [
        "Bi",
        "##ko",
        "zi",
        "##ga",
        "pu",
        "##ku",
        "ise",
        "nai",
        "##ra",
        "n",
        "'",
        "aka",
        "##ụ",
        "##nt",
        "##ụ",
        "m"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.778
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Biko ziga puku ise naira n'akaụntụ m",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Bi",
        "ko",
        "▁",
        "ziga",
        "▁pu",
        "ku",
        "▁ise",
        "▁na",
        "ira",
        "▁n",
        "'",
        "aka",
        "ụ",
        "nt",
        "ụ",
        "▁m"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.778
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Ab",
        "eg",
        " transfer",
        " five",
        " thousand",
        " n",
        "aira",
        " to",
        " my",
        " account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Ab",
        "eg",
        " transfer",
        " five",
        " thousand",
        " na",
        "ira",
        " to",
        " my",
        " account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "Ab",
        "eg",
        "Ġtransfer",
        "Ġfive",
        "Ġthousand",
        "Ġn",
        "aira",
        "Ġto",
        "Ġmy",
        "Ġaccount"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "A",
        "beg",
        "▁transfer",
        "▁five",
        "▁thousand",
        "▁naira",
        "▁to",
        "▁my",
        "▁account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 8,
      "token_ratio_vs_english": 1.125
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Ab",
        "eg",
        "▁transfer",
        "▁five",
        "▁thousand",
        "▁n",
        "air",
        "a",
        "▁to",
        "▁my",
        "▁account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "Abe",
        "##g",
        "transfer",
        "five",
        "thousand",
        "nai",
        "##ra",
        "to",
        "my",
        "account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 84,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Abeg transfer five thousand naira to my account",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁A",
        "beg",
        "▁transfer",
        "▁five",
        "▁thousand",
        "▁na",
        "ira",
        "▁to",
        "▁my",
        "▁account"
      ],
      "english_equivalent": "Please transfer five thousand naira to my account",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "My",
        " bank",
        " app",
        " is",
        " not",
        " working",
        ",",
        " I",
        " cannot",
        " check",
        " my",
        " balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "My",
        " bank",
        " app",
        " is",
        " not",
        " working",
        ",",
        " I",
        " cannot",
        " check",
        " my",
        " balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "My",
        "Ġbank",
        "Ġapp",
        "Ġis",
        "Ġnot",
        "Ġworking",
        ",",
        "ĠI",
        "Ġcannot",
        "Ġcheck",
        "Ġmy",
        "Ġbalance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "My",
        "▁bank",
        "▁app",
        "▁is",
        "▁not",
        "▁working",
        ",",
        "▁I",
        "▁cannot",
        "▁check",
        "▁my",
        "▁balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁My",
        "▁bank",
        "▁app",
        "▁is",
        "▁not",
        "▁working",
        ",",
        "▁I",
        "▁cannot",
        "▁check",
        "▁my",
        "▁balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "My",
        "bank",
        "app",
        "is",
        "not",
        "working",
        ",",
        "I",
        "cannot",
        "check",
        "my",
        "balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "My bank app is not working, I cannot check my balance",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁My",
        "▁bank",
        "▁app",
        "▁is",
        "▁not",
        "▁working",
        ",",
        "▁I",
        "▁cannot",
        "▁check",
        "▁my",
        "▁balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 42,
      "token_splits": [
        "À",
        "p",
        "ù",
        " b",
        "á",
        "ń",
        "k",
        "ì",
        " mi",
        " k",
        "ò",
        " ",
        "ń",
        " �",
        "�",
        "�",
        "i",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́",
        ",",
        " è",
        "mi",
        " k",
        "ò",
        " l",
        "è",
        " �",
        "�",
        "�",
        "ày",
        "�",
        "�",
        "̀",
        "w",
        "ò",
        " ow",
        "ó",
        " mi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 26,
      "token_splits": [
        "À",
        "p",
        "ù",
        " bá",
        "ń",
        "k",
        "ì",
        " mi",
        " kò",
        " ń",
        " ṣiṣẹ",
        "́",
        ",",
        " è",
        "mi",
        " kò",
        " lè",
        " ṣ",
        "ày",
        "ẹ",
        "̀",
        "w",
        "ò",
        " ow",
        "ó",
        " mi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 38,
      "token_splits": [
        "ÃĢ",
        "p",
        "Ã¹",
        "Ġb",
        "Ã¡",
        "ÅĦ",
        "k",
        "Ã¬",
        "Ġmi",
        "Ġk",
        "Ã²",
        "Ġ",
        "ÅĦ",
        "Ġá",
        "¹",
        "£i",
        "á¹",
        "£",
        "áº¹",
        "Ìģ",
        ",",
        "ĠÃ¨",
        "mi",
        "Ġk",
        "Ã²",
        "Ġl",
        "Ã¨",
        "Ġá",
        "¹",
        "£",
        "Ãły",
        "áº¹",
        "ÌĢ",
        "w",
        "Ã²",
        "Ġow",
        "Ã³",
        "Ġmi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.167
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 35,
      "token_splits": [
        "À",
        "p",
        "ù",
        "▁bá",
        "ń",
        "k",
        "ì",
        "▁mi",
        "▁k",
        "ò",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "ẹ",
        "́",
        ",",
        "▁è",
        "mi",
        "▁k",
        "ò",
        "▁l",
        "è",
        "▁",
        "ṣ",
        "ày",
        "ẹ",
        "̀",
        "w",
        "ò",
        "▁ow",
        "ó",
        "▁mi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.917
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 42,
      "token_splits": [
        "▁À",
        "p",
        "ù",
        "▁b",
        "á",
        "ń",
        "k",
        "ì",
        "▁mi",
        "▁k",
        "ò",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        ",",
        "▁è",
        "mi",
        "▁k",
        "ò",
        "▁l",
        "è",
        "▁",
        "ṣ",
        "à",
        "y",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "w",
        "ò",
        "▁o",
        "w",
        "ó",
        "▁mi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.5
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 30,
      "token_splits": [
        "À",
        "##p",
        "##ù",
        "bá",
        "##ń",
        "##k",
        "##ì",
        "mi",
        "kò",
        "ń",
        "ṣ",
        "##i",
        "##ṣ",
        "##ẹ́",
        ",",
        "è",
        "##mi",
        "kò",
        "l",
        "##è",
        "ṣ",
        "##à",
        "##y",
        "##ẹ̀",
        "##w",
        "##ò",
        "o",
        "##w",
        "##ó",
        "mi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àpù báńkì mi kò ń ṣiṣẹ́, èmi kò lè ṣàyẹ̀wò owó mi",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 37,
      "token_splits": [
        "▁À",
        "p",
        "ù",
        "▁bá",
        "ń",
        "k",
        "ì",
        "▁mi",
        "▁k",
        "ò",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "i",
        "ṣ",
        "ẹ",
        "́",
        ",",
        "▁è",
        "mi",
        "▁k",
        "ò",
        "▁",
        "lè",
        "▁",
        "ṣ",
        "à",
        "y",
        "ẹ",
        "̀",
        "w",
        "ò",
        "▁",
        "ow",
        "ó",
        "▁mi"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.083
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "App",
        "in",
        " bank",
        "ina",
        " ba",
        " ya",
        " a",
        "iki",
        ",",
        " ba",
        " z",
        "an",
        " i",
        "ya",
        " dub",
        "a",
        " ku",
        "�",
        "�",
        "ina",
        " ba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "App",
        "in",
        " bank",
        "ina",
        " ba",
        " ya",
        " aiki",
        ",",
        " ba",
        " zan",
        " iya",
        " d",
        "uba",
        " ku",
        "ɗ",
        "ina",
        " ba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.417
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 21,
      "token_splits": [
        "App",
        "in",
        "Ġbank",
        "ina",
        "Ġba",
        "Ġya",
        "Ġa",
        "iki",
        ",",
        "Ġba",
        "Ġz",
        "an",
        "Ġi",
        "ya",
        "Ġdub",
        "a",
        "Ġku",
        "É",
        "Ĺ",
        "ina",
        "Ġba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "App",
        "in",
        "▁bank",
        "ina",
        "▁ba",
        "▁ya",
        "▁a",
        "iki",
        ",",
        "▁ba",
        "▁zan",
        "▁iya",
        "▁d",
        "uba",
        "▁ku",
        "ɗ",
        "ina",
        "▁ba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁App",
        "in",
        "▁bank",
        "ina",
        "▁ba",
        "▁ya",
        "▁a",
        "iki",
        ",",
        "▁ba",
        "▁z",
        "an",
        "▁i",
        "ya",
        "▁dub",
        "a",
        "▁k",
        "u",
        "<0xC9>",
        "<0x97>",
        "ina",
        "▁ba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.833
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 18,
      "token_splits": [
        "App",
        "##in",
        "bank",
        "##ina",
        "ba",
        "ya",
        "ai",
        "##ki",
        ",",
        "ba",
        "za",
        "##n",
        "i",
        "##ya",
        "dub",
        "##a",
        "[UNK]",
        "ba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Appin bankina ba ya aiki, ba zan iya duba kuɗina ba",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 18,
      "token_splits": [
        "▁App",
        "in",
        "▁bank",
        "ina",
        "▁ba",
        "▁ya",
        "▁aiki",
        ",",
        "▁ba",
        "▁za",
        "n",
        "▁iya",
        "▁du",
        "ba",
        "▁ku",
        "ɗ",
        "ina",
        "▁ba"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 32,
      "token_splits": [
        "Ng",
        "wa",
        " ",
        "ụ",
        "l",
        "ọ",
        " ak",
        "ụ",
        " m",
        " an",
        "agh",
        "ị",
        " ar",
        "ụ",
        " ",
        "ọ",
        "r",
        "ụ",
        ",",
        " en",
        "weg",
        "h",
        "ị",
        " m",
        " i",
        "ke",
        " ",
        "ị",
        "le",
        "le",
        " ego",
        " m"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 16,
      "token_splits": [
        "Ng",
        "wa",
        " ụlọ",
        " akụ",
        " m",
        " anaghị",
        " arụ",
        " ọrụ",
        ",",
        " enweghị",
        " m",
        " ike",
        " ị",
        "lele",
        " ego",
        " m"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 32,
      "token_splits": [
        "Ng",
        "wa",
        "Ġ",
        "á»¥",
        "l",
        "á»į",
        "Ġak",
        "á»¥",
        "Ġm",
        "Ġan",
        "agh",
        "á»ĭ",
        "Ġar",
        "á»¥",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        ",",
        "Ġen",
        "weg",
        "h",
        "á»ĭ",
        "Ġm",
        "Ġi",
        "ke",
        "Ġ",
        "á»ĭ",
        "le",
        "le",
        "Ġego",
        "Ġm"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 29,
      "token_splits": [
        "Ng",
        "wa",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ",
        "▁m",
        "▁ana",
        "gh",
        "ị",
        "▁ar",
        "ụ",
        "▁ọ",
        "r",
        "ụ",
        ",",
        "▁en",
        "we",
        "gh",
        "ị",
        "▁m",
        "▁ike",
        "▁",
        "ị",
        "lele",
        "▁ego",
        "▁m"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.417
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 32,
      "token_splits": [
        "▁Ng",
        "wa",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ",
        "▁m",
        "▁an",
        "agh",
        "ị",
        "▁ar",
        "ụ",
        "▁",
        "ọ",
        "r",
        "ụ",
        ",",
        "▁en",
        "we",
        "gh",
        "ị",
        "▁m",
        "▁i",
        "ke",
        "▁",
        "ị",
        "le",
        "le",
        "▁ego",
        "▁m"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 27,
      "token_splits": [
        "Ng",
        "##wa",
        "ụ",
        "##l",
        "##ọ",
        "ak",
        "##ụ",
        "m",
        "ana",
        "##gh",
        "##ị",
        "ar",
        "##ụ",
        "ọ",
        "##r",
        "##ụ",
        ",",
        "enw",
        "##eg",
        "##hị",
        "m",
        "ik",
        "##e",
        "ị",
        "##lele",
        "ego",
        "m"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ngwa ụlọ akụ m anaghị arụ ọrụ, enweghị m ike ịlele ego m",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 31,
      "token_splits": [
        "▁Ng",
        "wa",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ak",
        "ụ",
        "▁m",
        "▁an",
        "agh",
        "ị",
        "▁ar",
        "ụ",
        "▁",
        "ọ",
        "r",
        "ụ",
        ",",
        "▁en",
        "weg",
        "h",
        "ị",
        "▁m",
        "▁i",
        "ke",
        "▁",
        "ị",
        "lele",
        "▁ego",
        "▁m"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.583
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "My",
        " bank",
        " app",
        " no",
        " de",
        "y",
        " work",
        ",",
        " I",
        " no",
        " fit",
        " check",
        " my",
        " balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "My",
        " bank",
        " app",
        " no",
        " dey",
        " work",
        ",",
        " I",
        " no",
        " fit",
        " check",
        " my",
        " balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "My",
        "Ġbank",
        "Ġapp",
        "Ġno",
        "Ġde",
        "y",
        "Ġwork",
        ",",
        "ĠI",
        "Ġno",
        "Ġfit",
        "Ġcheck",
        "Ġmy",
        "Ġbalance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "My",
        "▁bank",
        "▁app",
        "▁no",
        "▁dey",
        "▁work",
        ",",
        "▁I",
        "▁no",
        "▁fit",
        "▁check",
        "▁my",
        "▁balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.083
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁My",
        "▁bank",
        "▁app",
        "▁no",
        "▁de",
        "y",
        "▁work",
        ",",
        "▁I",
        "▁no",
        "▁fit",
        "▁check",
        "▁my",
        "▁balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "My",
        "bank",
        "app",
        "no",
        "de",
        "##y",
        "work",
        ",",
        "I",
        "no",
        "fit",
        "check",
        "my",
        "balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 85,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My bank app no dey work, I no fit check my balance",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁My",
        "▁bank",
        "▁app",
        "▁no",
        "▁de",
        "y",
        "▁work",
        ",",
        "▁I",
        "▁no",
        "▁fit",
        "▁check",
        "▁my",
        "▁balance"
      ],
      "english_equivalent": "My bank app is not working, I cannot check my balance",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "The",
        " exchange",
        " rate",
        " for",
        " dollars",
        " to",
        " n",
        "aira",
        " keeps",
        " increasing",
        " every",
        " day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "The",
        " exchange",
        " rate",
        " for",
        " dollars",
        " to",
        " na",
        "ira",
        " keeps",
        " increasing",
        " every",
        " day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "Ġexchange",
        "Ġrate",
        "Ġfor",
        "Ġdollars",
        "Ġto",
        "Ġn",
        "aira",
        "Ġkeeps",
        "Ġincreasing",
        "Ġevery",
        "Ġday"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "▁exchange",
        "▁rate",
        "▁for",
        "▁dollars",
        "▁to",
        "▁naira",
        "▁keeps",
        "▁increasing",
        "▁every",
        "▁day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁The",
        "▁exchange",
        "▁rate",
        "▁for",
        "▁dollars",
        "▁to",
        "▁n",
        "air",
        "a",
        "▁keeps",
        "▁increasing",
        "▁every",
        "▁day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "exchange",
        "rate",
        "for",
        "dollars",
        "to",
        "nai",
        "##ra",
        "keeps",
        "increasing",
        "every",
        "day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "eng",
      "language_name": "English",
      "text": "The exchange rate for dollars to naira keeps increasing every day",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁The",
        "▁exchange",
        "▁rate",
        "▁for",
        "▁dollars",
        "▁to",
        "▁na",
        "ira",
        "▁keep",
        "s",
        "▁increasing",
        "▁every",
        "▁day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 30,
      "token_splits": [
        "I",
        "ye",
        " ow",
        "ó",
        " d",
        "ọ",
        "́",
        "l",
        "à",
        " sí",
        " n",
        "á",
        "í",
        "r",
        "à",
        " ",
        "ń",
        " p",
        "ọ",
        "̀",
        " sí",
        " i",
        " l",
        "ój",
        "oo",
        "j",
        "ú",
        "m",
        "ọ",
        "́"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.5
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 24,
      "token_splits": [
        "I",
        "ye",
        " ow",
        "ó",
        " d",
        "ọ",
        "́",
        "l",
        "à",
        " sí",
        " ná",
        "ír",
        "à",
        " ń",
        " pọ",
        "̀",
        " sí",
        " i",
        " l",
        "ój",
        "oo",
        "jú",
        "mọ",
        "́"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 28,
      "token_splits": [
        "I",
        "ye",
        "Ġow",
        "Ã³",
        "Ġd",
        "á»į",
        "Ìģ",
        "l",
        "Ãł",
        "ĠsÃŃ",
        "ĠnÃ¡",
        "ÃŃr",
        "Ãł",
        "Ġ",
        "ÅĦ",
        "Ġp",
        "á»į",
        "ÌĢ",
        "ĠsÃŃ",
        "Ġi",
        "Ġl",
        "Ã³j",
        "oo",
        "j",
        "Ãº",
        "m",
        "á»į",
        "Ìģ"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 26,
      "token_splits": [
        "I",
        "ye",
        "▁ow",
        "ó",
        "▁d",
        "ọ",
        "́",
        "là",
        "▁sí",
        "▁ná",
        "ír",
        "à",
        "▁",
        "ń",
        "▁p",
        "ọ",
        "̀",
        "▁sí",
        "▁i",
        "▁l",
        "ój",
        "oo",
        "jú",
        "m",
        "ọ",
        "́"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.364
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 29,
      "token_splits": [
        "▁I",
        "ye",
        "▁o",
        "w",
        "ó",
        "▁d",
        "ọ",
        "́",
        "l",
        "à",
        "▁sí",
        "▁ná",
        "ír",
        "à",
        "▁",
        "ń",
        "▁p",
        "ọ",
        "̀",
        "▁sí",
        "▁i",
        "▁l",
        "ój",
        "oo",
        "j",
        "ú",
        "m",
        "ọ",
        "́"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.231
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 23,
      "token_splits": [
        "I",
        "##ye",
        "o",
        "##w",
        "##ó",
        "d",
        "##ọ́",
        "##là",
        "sí",
        "ná",
        "##ír",
        "##à",
        "ń",
        "p",
        "##ọ̀",
        "sí",
        "i",
        "l",
        "##ój",
        "##oo",
        "##jú",
        "##m",
        "##ọ́"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.917
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iye owó dọ́là sí náírà ń pọ̀ sí i lójoojúmọ́",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 27,
      "token_splits": [
        "▁I",
        "ye",
        "▁",
        "ow",
        "ó",
        "▁d",
        "ọ",
        "́",
        "là",
        "▁sí",
        "▁ná",
        "í",
        "rà",
        "▁",
        "ń",
        "▁p",
        "ọ",
        "̀",
        "▁sí",
        "▁i",
        "▁ló",
        "jo",
        "o",
        "jú",
        "m",
        "ọ",
        "́"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.077
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 23,
      "token_splits": [
        "Far",
        "ash",
        "in",
        " mus",
        "ayar",
        " d",
        "ala",
        " zu",
        "wa",
        " n",
        "aira",
        " y",
        "ana",
        " ",
        "�",
        "�",
        "aru",
        "wa",
        " k",
        "ow",
        "ace",
        " r",
        "ana"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.917
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Far",
        "ashin",
        " mus",
        "ayar",
        " dala",
        " zuwa",
        " na",
        "ira",
        " yana",
        " ƙ",
        "aru",
        "wa",
        " kow",
        "ace",
        " rana"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 22,
      "token_splits": [
        "Far",
        "ash",
        "in",
        "Ġmus",
        "ayar",
        "Ġd",
        "ala",
        "Ġzu",
        "wa",
        "Ġn",
        "aira",
        "Ġyana",
        "Ġ",
        "Æ",
        "Ļ",
        "aru",
        "wa",
        "Ġk",
        "ow",
        "ace",
        "Ġr",
        "ana"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.833
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 16,
      "token_splits": [
        "Far",
        "ash",
        "in",
        "▁mus",
        "ayar",
        "▁dala",
        "▁zu",
        "wa",
        "▁naira",
        "▁yana",
        "▁ƙ",
        "aru",
        "wa",
        "▁kow",
        "ace",
        "▁rana"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.455
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 25,
      "token_splits": [
        "▁Far",
        "ash",
        "in",
        "▁mus",
        "ay",
        "ar",
        "▁d",
        "ala",
        "▁zu",
        "wa",
        "▁n",
        "air",
        "a",
        "▁y",
        "ana",
        "▁",
        "<0xC6>",
        "<0x99>",
        "aru",
        "wa",
        "▁k",
        "ow",
        "ace",
        "▁r",
        "ana"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.923
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 18,
      "token_splits": [
        "Far",
        "##ashi",
        "##n",
        "mu",
        "##say",
        "##ar",
        "dala",
        "zu",
        "##wa",
        "nai",
        "##ra",
        "yana",
        "[UNK]",
        "ko",
        "##wa",
        "##ce",
        "ran",
        "##a"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Farashin musayar dala zuwa naira yana ƙaruwa kowace rana",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 18,
      "token_splits": [
        "▁Fara",
        "shin",
        "▁mus",
        "a",
        "yar",
        "▁dala",
        "▁zuwa",
        "▁na",
        "ira",
        "▁yana",
        "▁",
        "ƙ",
        "ar",
        "uwa",
        "▁",
        "kowa",
        "ce",
        "▁rana"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.385
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 26,
      "token_splits": [
        "�",
        "�",
        "n",
        "ụ",
        " ego",
        " mg",
        "ban",
        "we",
        " d",
        "ọ",
        "la",
        " na",
        " n",
        "aira",
        " na",
        "-ab",
        "aw",
        "anye",
        " k",
        "wa",
        " ",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "Ọ",
        "n",
        "ụ",
        " ego",
        " mgbanwe",
        " d",
        "ọ",
        "la",
        " na",
        " na",
        "ira",
        " na",
        "-ab",
        "aw",
        "anye",
        " kwa",
        " ụbọchị"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.417
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 26,
      "token_splits": [
        "á»",
        "Į",
        "n",
        "á»¥",
        "Ġego",
        "Ġmg",
        "ban",
        "we",
        "Ġd",
        "á»į",
        "la",
        "Ġna",
        "Ġn",
        "aira",
        "Ġna",
        "-ab",
        "aw",
        "anye",
        "Ġk",
        "wa",
        "Ġ",
        "á»¥",
        "b",
        "á»į",
        "ch",
        "á»ĭ"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 24,
      "token_splits": [
        "Ọ",
        "n",
        "ụ",
        "▁ego",
        "▁mg",
        "ban",
        "we",
        "▁d",
        "ọ",
        "la",
        "▁na",
        "▁naira",
        "▁na",
        "-",
        "ab",
        "aw",
        "anye",
        "▁kwa",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.182
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 31,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "n",
        "ụ",
        "▁ego",
        "▁mg",
        "ban",
        "we",
        "▁d",
        "ọ",
        "la",
        "▁na",
        "▁n",
        "air",
        "a",
        "▁na",
        "-",
        "ab",
        "aw",
        "any",
        "e",
        "▁k",
        "wa",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.385
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 23,
      "token_splits": [
        "Ọ",
        "##n",
        "##ụ",
        "ego",
        "mg",
        "##ban",
        "##we",
        "d",
        "##ọ",
        "##la",
        "na",
        "nai",
        "##ra",
        "na",
        "-",
        "ab",
        "##awan",
        "##ye",
        "kwa",
        "ụ",
        "##b",
        "##ọc",
        "##hị"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.917
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ego mgbanwe dọla na naira na-abawanye kwa ụbọchị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 26,
      "token_splits": [
        "▁",
        "Ọ",
        "n",
        "ụ",
        "▁ego",
        "▁mg",
        "ban",
        "we",
        "▁d",
        "ọ",
        "la",
        "▁na",
        "▁na",
        "ira",
        "▁na",
        "-",
        "aba",
        "wany",
        "e",
        "▁kwa",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "D",
        "ollar",
        " to",
        " n",
        "aira",
        " rate",
        " de",
        "y",
        " go",
        " up",
        " every",
        " day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Dollar",
        " to",
        " na",
        "ira",
        " rate",
        " dey",
        " go",
        " up",
        " every",
        " day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "D",
        "ollar",
        "Ġto",
        "Ġn",
        "aira",
        "Ġrate",
        "Ġde",
        "y",
        "Ġgo",
        "Ġup",
        "Ġevery",
        "Ġday"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "Dollar",
        "▁to",
        "▁naira",
        "▁rate",
        "▁dey",
        "▁go",
        "▁up",
        "▁every",
        "▁day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 11,
      "token_ratio_vs_english": 0.818
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁D",
        "ollar",
        "▁to",
        "▁n",
        "air",
        "a",
        "▁rate",
        "▁de",
        "y",
        "▁go",
        "▁up",
        "▁every",
        "▁day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Dollar",
        "to",
        "nai",
        "##ra",
        "rate",
        "de",
        "##y",
        "go",
        "up",
        "every",
        "day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 12,
      "token_ratio_vs_english": 0.917
    },
    {
      "entry_id": 86,
      "category": "fintech",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Dollar to naira rate dey go up every day",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Dollar",
        "▁to",
        "▁na",
        "ira",
        "▁rate",
        "▁de",
        "y",
        "▁go",
        "▁up",
        "▁every",
        "▁day"
      ],
      "english_equivalent": "The exchange rate for dollars to naira keeps increasing every day",
      "english_tokens": 13,
      "token_ratio_vs_english": 0.846
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "The",
        " mas",
        "quer",
        "ade",
        " festival",
        " is",
        " a",
        " sacred",
        " tradition",
        " in",
        " our",
        " community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "The",
        " masquer",
        "ade",
        " festival",
        " is",
        " a",
        " sacred",
        " tradition",
        " in",
        " our",
        " community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "Ġmas",
        "quer",
        "ade",
        "Ġfestival",
        "Ġis",
        "Ġa",
        "Ġsacred",
        "Ġtradition",
        "Ġin",
        "Ġour",
        "Ġcommunity"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "The",
        "▁masquerade",
        "▁festival",
        "▁is",
        "▁a",
        "▁sacred",
        "▁tradition",
        "▁in",
        "▁our",
        "▁community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁The",
        "▁mas",
        "quer",
        "ade",
        "▁festival",
        "▁is",
        "▁a",
        "▁sacred",
        "▁tradition",
        "▁in",
        "▁our",
        "▁community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 12,
      "token_splits": [
        "The",
        "mas",
        "##quera",
        "##de",
        "festival",
        "is",
        "a",
        "sacred",
        "tradition",
        "in",
        "our",
        "community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The masquerade festival is a sacred tradition in our community",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁The",
        "▁mas",
        "que",
        "rade",
        "▁festival",
        "▁is",
        "▁a",
        "▁sa",
        "cred",
        "▁tradition",
        "▁in",
        "▁our",
        "▁community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 27,
      "token_splits": [
        "Od",
        "ún",
        " eg",
        "ú",
        "ng",
        "ún",
        " j",
        "�",
        "�",
        "́",
        " à",
        "�",
        "�",
        "�",
        "à",
        " m",
        "ím",
        "ọ",
        "́",
        " n",
        "ín",
        "ú",
        " �",
        "�",
        "l",
        "ú",
        " wa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.25
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 21,
      "token_splits": [
        "Od",
        "ún",
        " eg",
        "úng",
        "ún",
        " jẹ",
        "́",
        " à",
        "ṣ",
        "à",
        " m",
        "ím",
        "ọ",
        "́",
        " n",
        "ín",
        "ú",
        " ì",
        "l",
        "ú",
        " wa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.909
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 24,
      "token_splits": [
        "Od",
        "Ãºn",
        "Ġeg",
        "Ãºng",
        "Ãºn",
        "Ġj",
        "áº¹",
        "Ìģ",
        "ĠÃł",
        "á¹",
        "£",
        "Ãł",
        "Ġm",
        "ÃŃm",
        "á»į",
        "Ìģ",
        "Ġn",
        "ÃŃn",
        "Ãº",
        "ĠÃ",
        "¬",
        "l",
        "Ãº",
        "Ġwa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 22,
      "token_splits": [
        "Od",
        "ún",
        "▁eg",
        "úng",
        "ún",
        "▁j",
        "ẹ",
        "́",
        "▁à",
        "ṣ",
        "à",
        "▁m",
        "ím",
        "ọ",
        "́",
        "▁n",
        "ín",
        "ú",
        "▁ì",
        "l",
        "ú",
        "▁wa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.2
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁Od",
        "ún",
        "▁eg",
        "ú",
        "ng",
        "ún",
        "▁j",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁à",
        "ṣ",
        "à",
        "▁m",
        "ím",
        "ọ",
        "́",
        "▁n",
        "ín",
        "ú",
        "▁",
        "ì",
        "l",
        "ú",
        "▁wa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.167
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 17,
      "token_splits": [
        "Od",
        "##ún",
        "e",
        "##gún",
        "##gún",
        "jẹ́",
        "à",
        "##ṣ",
        "##à",
        "mí",
        "##m",
        "##ọ́",
        "ní",
        "##nú",
        "ì",
        "##lú",
        "wa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.417
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Odún egúngún jẹ́ àṣà mímọ́ nínú ìlú wa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 21,
      "token_splits": [
        "▁Od",
        "ún",
        "▁e",
        "gún",
        "gún",
        "▁j",
        "ẹ",
        "́",
        "▁à",
        "ṣ",
        "à",
        "▁mí",
        "m",
        "ọ",
        "́",
        "▁ní",
        "nú",
        "▁",
        "ì",
        "lú",
        "▁wa"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.615
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 24,
      "token_splits": [
        "B",
        "ik",
        "in",
        " ",
        "�",
        "�",
        "an",
        " k",
        "all",
        "abi",
        " al",
        "'",
        "ada",
        " ce",
        " mai",
        " ts",
        "ark",
        "i",
        " a",
        " al",
        "'",
        "umm",
        "arm",
        "u"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 19,
      "token_splits": [
        "B",
        "ikin",
        " ɗ",
        "an",
        " kall",
        "abi",
        " al",
        "'",
        "ada",
        " ce",
        " mai",
        " ts",
        "arki",
        " a",
        " al",
        "'",
        "umm",
        "arm",
        "u"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.727
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 24,
      "token_splits": [
        "B",
        "ik",
        "in",
        "Ġ",
        "É",
        "Ĺ",
        "an",
        "Ġk",
        "all",
        "abi",
        "Ġal",
        "'",
        "ada",
        "Ġce",
        "Ġmai",
        "Ġts",
        "ark",
        "i",
        "Ġa",
        "Ġal",
        "'",
        "umm",
        "arm",
        "u"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 20,
      "token_splits": [
        "B",
        "ikin",
        "▁",
        "ɗ",
        "an",
        "▁kalla",
        "bi",
        "▁al",
        "'",
        "ada",
        "▁ce",
        "▁mai",
        "▁ts",
        "arki",
        "▁a",
        "▁al",
        "'",
        "um",
        "mar",
        "mu"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 23,
      "token_splits": [
        "▁B",
        "ik",
        "in",
        "▁",
        "<0xC9>",
        "<0x97>",
        "an",
        "▁kall",
        "abi",
        "▁al",
        "'",
        "ada",
        "▁ce",
        "▁mai",
        "▁ts",
        "ark",
        "i",
        "▁a",
        "▁al",
        "'",
        "umm",
        "arm",
        "u"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.917
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 18,
      "token_splits": [
        "Bi",
        "##kin",
        "[UNK]",
        "kalla",
        "##bi",
        "al",
        "'",
        "ada",
        "ce",
        "mai",
        "tsar",
        "##ki",
        "a",
        "al",
        "'",
        "um",
        "##mar",
        "##mu"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin ɗan kallabi al'ada ce mai tsarki a al'ummarmu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 19,
      "token_splits": [
        "▁Bikin",
        "▁",
        "ɗ",
        "an",
        "▁kalla",
        "bi",
        "▁al",
        "'",
        "ada",
        "▁ce",
        "▁mai",
        "▁tsa",
        "rki",
        "▁a",
        "▁al",
        "'",
        "um",
        "mar",
        "mu"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.462
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "Em",
        "ume",
        " m",
        "man",
        "w",
        "ụ",
        " b",
        "ụ",
        " o",
        "men",
        "ala",
        " d",
        "ị",
        " ns",
        "ọ",
        " n",
        "'",
        "ob",
        "odo",
        " any",
        "ị"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Em",
        "ume",
        " m",
        "man",
        "wụ",
        " bụ",
        " omen",
        "ala",
        " dị",
        " ns",
        "ọ",
        " n",
        "'ob",
        "odo",
        " anyị"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.364
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 19,
      "token_splits": [
        "Em",
        "ume",
        "Ġm",
        "man",
        "w",
        "á»¥",
        "Ġbá»¥",
        "Ġo",
        "men",
        "ala",
        "Ġdá»ĭ",
        "Ġns",
        "á»į",
        "Ġn",
        "'",
        "ob",
        "odo",
        "Ġany",
        "á»ĭ"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.583
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "Emu",
        "me",
        "▁m",
        "man",
        "w",
        "ụ",
        "▁bụ",
        "▁omen",
        "ala",
        "▁dị",
        "▁ns",
        "ọ",
        "▁n",
        "'",
        "ob",
        "odo",
        "▁any",
        "ị"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.8
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 21,
      "token_splits": [
        "▁Em",
        "ume",
        "▁m",
        "man",
        "w",
        "ụ",
        "▁b",
        "ụ",
        "▁o",
        "men",
        "ala",
        "▁d",
        "ị",
        "▁ns",
        "ọ",
        "▁n",
        "'",
        "ob",
        "odo",
        "▁any",
        "ị"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "Em",
        "##ume",
        "mm",
        "##an",
        "##w",
        "##ụ",
        "b",
        "##ụ",
        "om",
        "##enal",
        "##a",
        "d",
        "##ị",
        "ns",
        "##ọ",
        "n",
        "'",
        "ob",
        "##odo",
        "any",
        "##ị"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume mmanwụ bụ omenala dị nsọ n'obodo anyị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 20,
      "token_splits": [
        "▁Em",
        "ume",
        "▁",
        "mman",
        "w",
        "ụ",
        "▁b",
        "ụ",
        "▁omen",
        "ala",
        "▁dị",
        "▁",
        "ns",
        "ọ",
        "▁n",
        "'",
        "obo",
        "do",
        "▁any",
        "ị"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.538
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Mas",
        "quer",
        "ade",
        " festival",
        " na",
        " holy",
        " tradition",
        " for",
        " our",
        " community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Mas",
        "quer",
        "ade",
        " festival",
        " na",
        " holy",
        " tradition",
        " for",
        " our",
        " community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 11,
      "token_ratio_vs_english": 0.909
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "Mas",
        "quer",
        "ade",
        "Ġfestival",
        "Ġna",
        "Ġholy",
        "Ġtradition",
        "Ġfor",
        "Ġour",
        "Ġcommunity"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "Mas",
        "querade",
        "▁festival",
        "▁na",
        "▁holy",
        "▁tradition",
        "▁for",
        "▁our",
        "▁community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 10,
      "token_ratio_vs_english": 0.9
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁Mas",
        "quer",
        "ade",
        "▁festival",
        "▁na",
        "▁holy",
        "▁tradition",
        "▁for",
        "▁our",
        "▁community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "Mas",
        "##quera",
        "##de",
        "festival",
        "na",
        "holy",
        "tradition",
        "for",
        "our",
        "community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 12,
      "token_ratio_vs_english": 0.833
    },
    {
      "entry_id": 87,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Masquerade festival na holy tradition for our community",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Mas",
        "que",
        "rade",
        "▁festival",
        "▁na",
        "▁ho",
        "ly",
        "▁tradition",
        "▁for",
        "▁our",
        "▁community"
      ],
      "english_equivalent": "The masquerade festival is a sacred tradition in our community",
      "english_tokens": 13,
      "token_ratio_vs_english": 0.846
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "The",
        " naming",
        " ceremony",
        " will",
        " hold",
        " on",
        " Saturday",
        " at",
        " our",
        " family",
        " house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "The",
        " naming",
        " ceremony",
        " will",
        " hold",
        " on",
        " Saturday",
        " at",
        " our",
        " family",
        " house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "Ġnaming",
        "Ġceremony",
        "Ġwill",
        "Ġhold",
        "Ġon",
        "ĠSaturday",
        "Ġat",
        "Ġour",
        "Ġfamily",
        "Ġhouse"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "▁naming",
        "▁ceremony",
        "▁will",
        "▁hold",
        "▁on",
        "▁Saturday",
        "▁at",
        "▁our",
        "▁family",
        "▁house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁The",
        "▁n",
        "aming",
        "▁ceremony",
        "▁will",
        "▁hold",
        "▁on",
        "▁Saturday",
        "▁at",
        "▁our",
        "▁family",
        "▁house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "naming",
        "ceremony",
        "will",
        "hold",
        "on",
        "Saturday",
        "at",
        "our",
        "family",
        "house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "eng",
      "language_name": "English",
      "text": "The naming ceremony will hold on Saturday at our family house",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁The",
        "▁naming",
        "▁ce",
        "remo",
        "ny",
        "▁will",
        "▁hold",
        "▁on",
        "▁Saturday",
        "▁at",
        "▁our",
        "▁family",
        "▁house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 45,
      "token_splits": [
        "Ay",
        "�",
        "�",
        "y",
        "�",
        "�",
        " �",
        "�",
        "s",
        "ọ",
        "m",
        "ọ",
        "l",
        "ór",
        "ú",
        "k",
        "ọ",
        " y",
        "ó",
        "ò",
        " w",
        "á",
        "y",
        "é",
        " n",
        "í",
        " ",
        "ọ",
        "j",
        "ọ",
        "́",
        " S",
        "át",
        "id",
        "é",
        " n",
        "í",
        " il",
        "é",
        " �",
        "�",
        "d",
        "í",
        "lé",
        " wa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 4.091
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 32,
      "token_splits": [
        "Ay",
        "ẹ",
        "y",
        "ẹ",
        " ì",
        "sọ",
        "mọ",
        "l",
        "ór",
        "ú",
        "kọ",
        " y",
        "ó",
        "ò",
        " w",
        "á",
        "y",
        "é",
        " ní",
        " ọjọ",
        "́",
        " S",
        "át",
        "idé",
        " ní",
        " il",
        "é",
        " ì",
        "d",
        "í",
        "lé",
        " wa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.909
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 39,
      "token_splits": [
        "Ay",
        "áº¹",
        "y",
        "áº¹",
        "ĠÃ",
        "¬",
        "s",
        "á»į",
        "m",
        "á»į",
        "l",
        "Ã³r",
        "Ãº",
        "k",
        "á»į",
        "Ġy",
        "Ã³",
        "Ã²",
        "Ġw",
        "Ã¡y",
        "Ã©",
        "ĠnÃŃ",
        "Ġ",
        "á»į",
        "j",
        "á»į",
        "Ìģ",
        "ĠS",
        "Ã¡t",
        "id",
        "Ã©",
        "ĠnÃŃ",
        "Ġil",
        "Ã©",
        "ĠÃ",
        "¬",
        "dÃŃ",
        "lÃ©",
        "Ġwa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.545
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 35,
      "token_splits": [
        "Ay",
        "ẹ",
        "y",
        "ẹ",
        "▁ì",
        "s",
        "ọ",
        "m",
        "ọ",
        "l",
        "ór",
        "ú",
        "k",
        "ọ",
        "▁y",
        "ó",
        "ò",
        "▁w",
        "áy",
        "é",
        "▁ní",
        "▁ọ",
        "j",
        "ọ",
        "́",
        "▁Sá",
        "tid",
        "é",
        "▁ní",
        "▁il",
        "é",
        "▁ì",
        "dí",
        "lé",
        "▁wa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.182
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 47,
      "token_splits": [
        "▁Ay",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "y",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "▁",
        "ì",
        "s",
        "ọ",
        "m",
        "ọ",
        "l",
        "ór",
        "ú",
        "k",
        "ọ",
        "▁y",
        "ó",
        "ò",
        "▁w",
        "á",
        "y",
        "é",
        "▁n",
        "í",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁S",
        "át",
        "id",
        "é",
        "▁n",
        "í",
        "▁il",
        "é",
        "▁",
        "ì",
        "d",
        "í",
        "lé",
        "▁wa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.917
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 33,
      "token_splits": [
        "Ay",
        "##ẹ",
        "##y",
        "##ẹ",
        "ì",
        "##s",
        "##ọ",
        "##m",
        "##ọ",
        "##ló",
        "##rú",
        "##k",
        "##ọ",
        "y",
        "##ó",
        "##ò",
        "w",
        "##áy",
        "##é",
        "ní",
        "ọ",
        "##j",
        "##ọ́",
        "Sá",
        "##tid",
        "##é",
        "ní",
        "il",
        "##é",
        "ì",
        "##dí",
        "##lé",
        "wa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ayẹyẹ ìsọmọlórúkọ yóò wáyé ní ọjọ́ Sátidé ní ilé ìdílé wa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 38,
      "token_splits": [
        "▁Ay",
        "ẹ",
        "y",
        "ẹ",
        "▁",
        "ì",
        "s",
        "ọ",
        "m",
        "ọ",
        "ló",
        "rú",
        "k",
        "ọ",
        "▁y",
        "ó",
        "ò",
        "▁w",
        "á",
        "y",
        "é",
        "▁ní",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁S",
        "át",
        "idé",
        "▁ní",
        "▁i",
        "lé",
        "▁",
        "ì",
        "dí",
        "lé",
        "▁wa"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.923
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 20,
      "token_splits": [
        "B",
        "ik",
        "in",
        " sun",
        "a",
        " z",
        "ai",
        " kas",
        "ance",
        " ran",
        "ar",
        " As",
        "abar",
        " a",
        " gid",
        "an",
        " i",
        "yal",
        "in",
        "mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.818
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "B",
        "ikin",
        " suna",
        " zai",
        " kasance",
        " ranar",
        " As",
        "abar",
        " a",
        " gidan",
        " iy",
        "alin",
        "mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.182
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 20,
      "token_splits": [
        "B",
        "ik",
        "in",
        "Ġsun",
        "a",
        "Ġz",
        "ai",
        "Ġkas",
        "ance",
        "Ġran",
        "ar",
        "ĠAs",
        "abar",
        "Ġa",
        "Ġgid",
        "an",
        "Ġi",
        "yal",
        "in",
        "mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.818
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "B",
        "ikin",
        "▁suna",
        "▁z",
        "ai",
        "▁kas",
        "ance",
        "▁ran",
        "ar",
        "▁As",
        "abar",
        "▁a",
        "▁g",
        "idan",
        "▁i",
        "yal",
        "in",
        "mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.636
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁B",
        "ik",
        "in",
        "▁sun",
        "a",
        "▁z",
        "ai",
        "▁k",
        "as",
        "ance",
        "▁ran",
        "ar",
        "▁As",
        "ab",
        "ar",
        "▁a",
        "▁g",
        "idan",
        "▁i",
        "yal",
        "in",
        "mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.833
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 19,
      "token_splits": [
        "Bi",
        "##kin",
        "sun",
        "##a",
        "za",
        "##i",
        "kas",
        "##ance",
        "ran",
        "##ar",
        "Asa",
        "##bar",
        "a",
        "gi",
        "##dan",
        "i",
        "##yal",
        "##in",
        "##mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.727
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Bikin suna zai kasance ranar Asabar a gidan iyalinmu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 12,
      "token_splits": [
        "▁Bikin",
        "▁suna",
        "▁zai",
        "▁kasance",
        "▁ranar",
        "▁Asa",
        "bar",
        "▁a",
        "▁gidan",
        "▁iya",
        "lin",
        "mu"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 13,
      "token_ratio_vs_english": 0.923
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 33,
      "token_splits": [
        "Em",
        "ume",
        " ",
        "ị",
        "g",
        "ụ",
        " a",
        "ha",
        " ga",
        "-",
        "eme",
        " n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        " Sat",
        "ọ",
        "dee",
        " n",
        "'",
        "ụ",
        "l",
        "ọ",
        " ez",
        "in",
        "ụ",
        "l",
        "ọ",
        " any",
        "ị"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 19,
      "token_splits": [
        "Em",
        "ume",
        " ị",
        "gụ",
        " aha",
        " ga",
        "-eme",
        " n",
        "'ụ",
        "bọchị",
        " Sat",
        "ọ",
        "dee",
        " n",
        "'ụ",
        "lọ",
        " ezin",
        "ụlọ",
        " anyị"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.727
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 33,
      "token_splits": [
        "Em",
        "ume",
        "Ġ",
        "á»ĭ",
        "g",
        "á»¥",
        "Ġa",
        "ha",
        "Ġga",
        "-",
        "eme",
        "Ġn",
        "'",
        "á»¥",
        "b",
        "á»į",
        "ch",
        "á»ĭ",
        "ĠSat",
        "á»į",
        "dee",
        "Ġn",
        "'",
        "á»¥",
        "l",
        "á»į",
        "Ġez",
        "in",
        "á»¥",
        "l",
        "á»į",
        "Ġany",
        "á»ĭ"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 32,
      "token_splits": [
        "Emu",
        "me",
        "▁",
        "ị",
        "g",
        "ụ",
        "▁aha",
        "▁ga",
        "-",
        "eme",
        "▁n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁Sat",
        "ọ",
        "dee",
        "▁n",
        "'",
        "ụ",
        "l",
        "ọ",
        "▁e",
        "zin",
        "ụ",
        "l",
        "ọ",
        "▁any",
        "ị"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.909
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 34,
      "token_splits": [
        "▁Em",
        "ume",
        "▁",
        "ị",
        "g",
        "ụ",
        "▁a",
        "ha",
        "▁ga",
        "-",
        "eme",
        "▁n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁Sat",
        "ọ",
        "de",
        "e",
        "▁n",
        "'",
        "ụ",
        "l",
        "ọ",
        "▁e",
        "zin",
        "ụ",
        "l",
        "ọ",
        "▁any",
        "ị"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.833
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 34,
      "token_splits": [
        "Em",
        "##ume",
        "ị",
        "##g",
        "##ụ",
        "ah",
        "##a",
        "ga",
        "-",
        "em",
        "##e",
        "n",
        "'",
        "ụ",
        "##b",
        "##ọc",
        "##hị",
        "Sa",
        "##t",
        "##ọ",
        "##de",
        "##e",
        "n",
        "'",
        "ụ",
        "##l",
        "##ọ",
        "ez",
        "##in",
        "##ụ",
        "##l",
        "##ọ",
        "any",
        "##ị"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.091
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Emume ịgụ aha ga-eme n'ụbọchị Satọdee n'ụlọ ezinụlọ anyị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 33,
      "token_splits": [
        "▁Em",
        "ume",
        "▁",
        "ị",
        "g",
        "ụ",
        "▁a",
        "ha",
        "▁ga",
        "-",
        "eme",
        "▁n",
        "'",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁Sat",
        "ọ",
        "de",
        "e",
        "▁n",
        "'",
        "ụ",
        "l",
        "ọ",
        "▁ezin",
        "ụ",
        "l",
        "ọ",
        "▁any",
        "ị"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.538
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "De",
        " naming",
        " ceremony",
        " go",
        " happen",
        " for",
        " Saturday",
        " for",
        " our",
        " family",
        " house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "De",
        " naming",
        " ceremony",
        " go",
        " happen",
        " for",
        " Saturday",
        " for",
        " our",
        " family",
        " house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "De",
        "Ġnaming",
        "Ġceremony",
        "Ġgo",
        "Ġhappen",
        "Ġfor",
        "ĠSaturday",
        "Ġfor",
        "Ġour",
        "Ġfamily",
        "Ġhouse"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "De",
        "▁naming",
        "▁ceremony",
        "▁go",
        "▁happen",
        "▁for",
        "▁Saturday",
        "▁for",
        "▁our",
        "▁family",
        "▁house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 12,
      "token_splits": [
        "▁De",
        "▁n",
        "aming",
        "▁ceremony",
        "▁go",
        "▁happen",
        "▁for",
        "▁Saturday",
        "▁for",
        "▁our",
        "▁family",
        "▁house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "De",
        "naming",
        "ceremony",
        "go",
        "happen",
        "for",
        "Saturday",
        "for",
        "our",
        "family",
        "house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 88,
      "category": "culture",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De naming ceremony go happen for Saturday for our family house",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁De",
        "▁naming",
        "▁ce",
        "remo",
        "ny",
        "▁go",
        "▁happen",
        "▁for",
        "▁Saturday",
        "▁for",
        "▁our",
        "▁family",
        "▁house"
      ],
      "english_equivalent": "The naming ceremony will hold on Saturday at our family house",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "We",
        " give",
        " thanks",
        " to",
        " God",
        " for",
        " this",
        " new",
        " day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "We",
        " give",
        " thanks",
        " to",
        " God",
        " for",
        " this",
        " new",
        " day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "We",
        "Ġgive",
        "Ġthanks",
        "Ġto",
        "ĠGod",
        "Ġfor",
        "Ġthis",
        "Ġnew",
        "Ġday"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "We",
        "▁give",
        "▁thanks",
        "▁to",
        "▁God",
        "▁for",
        "▁this",
        "▁new",
        "▁day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁We",
        "▁give",
        "▁thanks",
        "▁to",
        "▁God",
        "▁for",
        "▁this",
        "▁new",
        "▁day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "We",
        "give",
        "thanks",
        "to",
        "God",
        "for",
        "this",
        "new",
        "day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "eng",
      "language_name": "English",
      "text": "We give thanks to God for this new day",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁We",
        "▁give",
        "▁thanks",
        "▁to",
        "▁God",
        "▁for",
        "▁this",
        "▁new",
        "▁day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 33,
      "token_splits": [
        "A",
        " d",
        "ú",
        "p",
        "�",
        "�",
        "́",
        " l",
        "ọ",
        "́",
        "w",
        "ọ",
        "́",
        " ",
        "�",
        "�",
        "l",
        "ọ",
        "́",
        "run",
        " f",
        "ún",
        " ",
        "ọ",
        "j",
        "ọ",
        "́",
        " t",
        "unt",
        "un",
        " y",
        "ì",
        "í"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.667
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 20,
      "token_splits": [
        "A",
        " dú",
        "p",
        "ẹ",
        "́",
        " lọ",
        "́",
        "wọ",
        "́",
        " Ọ",
        "lọ",
        "́",
        "run",
        " fún",
        " ọjọ",
        "́",
        " tuntun",
        " y",
        "ì",
        "í"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.222
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 30,
      "token_splits": [
        "A",
        "Ġd",
        "Ãºp",
        "áº¹",
        "Ìģ",
        "Ġl",
        "á»į",
        "Ìģ",
        "w",
        "á»į",
        "Ìģ",
        "Ġá»",
        "Į",
        "l",
        "á»į",
        "Ìģ",
        "run",
        "Ġf",
        "Ãºn",
        "Ġ",
        "á»į",
        "j",
        "á»į",
        "Ìģ",
        "Ġt",
        "unt",
        "un",
        "Ġy",
        "Ã¬",
        "ÃŃ"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 27,
      "token_splits": [
        "A",
        "▁d",
        "úp",
        "ẹ",
        "́",
        "▁lọ",
        "́",
        "w",
        "ọ",
        "́",
        "▁",
        "Ọ",
        "l",
        "ọ",
        "́",
        "run",
        "▁f",
        "ún",
        "▁ọ",
        "j",
        "ọ",
        "́",
        "▁tunt",
        "un",
        "▁y",
        "ì",
        "í"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 35,
      "token_splits": [
        "▁A",
        "▁d",
        "ú",
        "p",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁l",
        "ọ",
        "́",
        "w",
        "ọ",
        "́",
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "l",
        "ọ",
        "́",
        "run",
        "▁f",
        "ún",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁t",
        "unt",
        "un",
        "▁y",
        "ì",
        "í"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.889
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "A",
        "d",
        "##úp",
        "##ẹ́",
        "l",
        "##ọ́",
        "##w",
        "##ọ́",
        "Ọ",
        "##l",
        "##ọ́",
        "##run",
        "f",
        "##ún",
        "ọ",
        "##j",
        "##ọ́",
        "tun",
        "##tun",
        "y",
        "##ìí"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "A dúpẹ́ lọ́wọ́ Ọlọ́run fún ọjọ́ tuntun yìí",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 29,
      "token_splits": [
        "▁A",
        "▁d",
        "úp",
        "ẹ",
        "́",
        "▁l",
        "ọ",
        "́",
        "w",
        "ọ",
        "́",
        "▁",
        "Ọ",
        "l",
        "ọ",
        "́",
        "run",
        "▁fú",
        "n",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁tuntu",
        "n",
        "▁y",
        "ì",
        "í"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.222
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "M",
        "una",
        " god",
        "iya",
        " ga",
        " Allah",
        " sab",
        "oda",
        " sab",
        "u",
        "war",
        " ran",
        "ar",
        " nan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "M",
        "una",
        " god",
        "iya",
        " ga",
        " Allah",
        " saboda",
        " sab",
        "uwar",
        " ranar",
        " nan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "M",
        "una",
        "Ġgod",
        "iya",
        "Ġga",
        "ĠAllah",
        "Ġsab",
        "oda",
        "Ġsab",
        "u",
        "war",
        "Ġran",
        "ar",
        "Ġnan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 14,
      "token_splits": [
        "M",
        "una",
        "▁go",
        "diya",
        "▁ga",
        "▁Allah",
        "▁sab",
        "oda",
        "▁sab",
        "u",
        "war",
        "▁ran",
        "ar",
        "▁nan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 16,
      "token_splits": [
        "▁M",
        "una",
        "▁god",
        "i",
        "ya",
        "▁ga",
        "▁All",
        "ah",
        "▁sab",
        "oda",
        "▁sab",
        "u",
        "war",
        "▁ran",
        "ar",
        "▁nan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.778
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "Mu",
        "##na",
        "god",
        "##iya",
        "ga",
        "Allah",
        "sa",
        "##bod",
        "##a",
        "sa",
        "##bu",
        "##war",
        "ran",
        "##ar",
        "nan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Muna godiya ga Allah saboda sabuwar ranar nan",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁Muna",
        "▁god",
        "iya",
        "▁ga",
        "▁Allah",
        "▁saboda",
        "▁s",
        "abuwar",
        "▁ranar",
        "▁nan"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 23,
      "token_splits": [
        "Any",
        "ị",
        " na",
        "-e",
        "ke",
        "le",
        " Ch",
        "uk",
        "wu",
        " maka",
        " ",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        " ",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        " a"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.556
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Any",
        "ị",
        " na",
        "-ek",
        "ele",
        " Ch",
        "ukwu",
        " maka",
        " ụbọchị",
        " ọhụrụ",
        " a"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 23,
      "token_splits": [
        "Any",
        "á»ĭ",
        "Ġna",
        "-e",
        "ke",
        "le",
        "ĠCh",
        "uk",
        "wu",
        "Ġmaka",
        "Ġ",
        "á»¥",
        "b",
        "á»į",
        "ch",
        "á»ĭ",
        "Ġ",
        "á»į",
        "h",
        "á»¥",
        "r",
        "á»¥",
        "Ġa"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.556
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 21,
      "token_splits": [
        "Any",
        "ị",
        "▁na",
        "-",
        "e",
        "kele",
        "▁Chuk",
        "wu",
        "▁maka",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁a"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁Any",
        "ị",
        "▁na",
        "-",
        "e",
        "ke",
        "le",
        "▁Ch",
        "uk",
        "w",
        "u",
        "▁m",
        "aka",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁a"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.889
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 20,
      "token_splits": [
        "Any",
        "##ị",
        "na",
        "-",
        "ek",
        "##ele",
        "Chu",
        "##k",
        "##wu",
        "maka",
        "ụ",
        "##b",
        "##ọc",
        "##hị",
        "ọ",
        "##h",
        "##ụ",
        "##r",
        "##ụ",
        "a"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.222
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Anyị na-ekele Chukwu maka ụbọchị ọhụrụ a",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 23,
      "token_splits": [
        "▁Any",
        "ị",
        "▁na",
        "-",
        "ek",
        "ele",
        "▁Chu",
        "kw",
        "u",
        "▁maka",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁a"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.556
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "We",
        " de",
        "y",
        " thank",
        " God",
        " for",
        " dis",
        " new",
        " day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 8,
      "token_splits": [
        "We",
        " dey",
        " thank",
        " God",
        " for",
        " dis",
        " new",
        " day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 0.889
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "We",
        "Ġde",
        "y",
        "Ġthank",
        "ĠGod",
        "Ġfor",
        "Ġdis",
        "Ġnew",
        "Ġday"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "We",
        "▁dey",
        "▁thank",
        "▁God",
        "▁for",
        "▁dis",
        "▁new",
        "▁day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 0.889
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁We",
        "▁de",
        "y",
        "▁thank",
        "▁God",
        "▁for",
        "▁dis",
        "▁new",
        "▁day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "We",
        "de",
        "##y",
        "than",
        "##k",
        "God",
        "for",
        "dis",
        "new",
        "day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 89,
      "category": "religion",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "We dey thank God for dis new day",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁We",
        "▁de",
        "y",
        "▁thank",
        "▁God",
        "▁for",
        "▁dis",
        "▁new",
        "▁day"
      ],
      "english_equivalent": "We give thanks to God for this new day",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "airport",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "p",
        "á",
        "p",
        "á",
        " ",
        "ọ",
        "k",
        "ọ",
        "̀",
        " �",
        "�",
        "fu",
        "ur",
        "uf",
        "ú"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 15.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "p",
        "áp",
        "á",
        " ọkọ",
        "̀",
        " ò",
        "f",
        "uur",
        "uf",
        "ú"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "p",
        "Ã¡p",
        "Ã¡",
        "Ġ",
        "á»į",
        "k",
        "á»į",
        "ÌĢ",
        "ĠÃ",
        "²",
        "fu",
        "ur",
        "uf",
        "Ãº"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 14.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "p",
        "áp",
        "á",
        "▁ọ",
        "k",
        "ọ",
        "̀",
        "▁ò",
        "fu",
        "ur",
        "uf",
        "ú"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 12.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁p",
        "á",
        "p",
        "á",
        "▁",
        "ọ",
        "k",
        "ọ",
        "̀",
        "▁",
        "ò",
        "f",
        "uur",
        "uf",
        "ú"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 15.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "p",
        "##áp",
        "##á",
        "ọ",
        "##k",
        "##ọ̀",
        "ò",
        "##fu",
        "##uru",
        "##f",
        "##ú"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 11.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "pápá ọkọ̀ òfuurufú",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 13,
      "token_splits": [
        "▁pá",
        "pá",
        "▁",
        "ọ",
        "k",
        "ọ",
        "̀",
        "▁",
        "ò",
        "fu",
        "u",
        "ruf",
        "ú"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 13.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "fil",
        "in",
        " j",
        "ir",
        "gin",
        " sama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "fil",
        "in",
        " jir",
        "gin",
        " sama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "fil",
        "in",
        "Ġj",
        "ir",
        "gin",
        "Ġsama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "fil",
        "in",
        "▁jir",
        "gin",
        "▁sama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁fil",
        "in",
        "▁j",
        "ir",
        "gin",
        "▁s",
        "ama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "fil",
        "##in",
        "ji",
        "##rgi",
        "##n",
        "sama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "filin jirgin sama",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 5,
      "token_splits": [
        "▁fili",
        "n",
        "▁jir",
        "gin",
        "▁sama"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "ọ",
        "d",
        "ụ",
        " ",
        "ụ",
        "gb",
        "ọ",
        " el",
        "u"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ọdụ",
        " ụgbọ",
        " elu"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "á»į",
        "d",
        "á»¥",
        "Ġ",
        "á»¥",
        "gb",
        "á»į",
        "Ġel",
        "u"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 8,
      "token_splits": [
        "ọ",
        "d",
        "ụ",
        "▁",
        "ụ",
        "gb",
        "ọ",
        "▁elu"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁",
        "ọ",
        "d",
        "ụ",
        "▁",
        "ụ",
        "gb",
        "ọ",
        "▁el",
        "u"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 8,
      "token_splits": [
        "ọ",
        "##d",
        "##ụ",
        "ụ",
        "##g",
        "##b",
        "##ọ",
        "elu"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọdụ ụgbọ elu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "ọ",
        "d",
        "ụ",
        "▁",
        "ụ",
        "gb",
        "ọ",
        "▁elu"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 90,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "airport",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁airport"
      ],
      "english_equivalent": "airport",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "telephone",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁telephone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "�",
        "�",
        "̀",
        "r",
        "ọ",
        " �",
        "�",
        "b",
        "án",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 15.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "ẹ",
        "̀",
        "rọ",
        " ì",
        "b",
        "án",
        "is",
        "ọ",
        "̀",
        "rọ",
        "̀"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 11.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "áº¹",
        "ÌĢ",
        "r",
        "á»į",
        "ĠÃ",
        "¬",
        "b",
        "Ã¡n",
        "is",
        "á»į",
        "ÌĢ",
        "r",
        "á»į",
        "ÌĢ"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 14.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁ì",
        "bán",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 12.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 17,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ọ",
        "▁",
        "ì",
        "b",
        "án",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 17.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "ẹ",
        "##̀",
        "##r",
        "##ọ",
        "ì",
        "##bán",
        "##is",
        "##ọ̀",
        "##r",
        "##ọ̀"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁",
        "ì",
        "bán",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 14.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 4,
      "token_splits": [
        "way",
        "ar",
        " tar",
        "ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "way",
        "ar",
        " tar",
        "ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 4,
      "token_splits": [
        "way",
        "ar",
        "Ġtar",
        "ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "way",
        "ar",
        "▁tar",
        "ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁way",
        "ar",
        "▁tar",
        "ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "way",
        "##ar",
        "tar",
        "##ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wayar tarho",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 4,
      "token_splits": [
        "▁way",
        "ar",
        "▁tar",
        "ho"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ek",
        "went",
        "ị"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ek",
        "went",
        "ị"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "ek",
        "went",
        "á»ĭ"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ek",
        "went",
        "ị"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁e",
        "kw",
        "ent",
        "ị"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 3,
      "token_splits": [
        "ek",
        "##went",
        "##ị"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ekwentị",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁ek",
        "went",
        "ị"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 91,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "phone",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁phone"
      ],
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "un",
        "iversity"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "un",
        "iversity"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "un",
        "iversity"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "university",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "il",
        "é",
        " ",
        "�",
        "�",
        "̀",
        "k",
        "ọ",
        "́",
        " g",
        "í",
        "ga"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 8,
      "token_splits": [
        "ilé",
        " ẹ",
        "̀",
        "kọ",
        "́",
        " g",
        "í",
        "ga"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "il",
        "Ã©",
        "Ġ",
        "áº¹",
        "ÌĢ",
        "k",
        "á»į",
        "Ìģ",
        "Ġg",
        "ÃŃ",
        "ga"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "ilé",
        "▁",
        "ẹ",
        "̀",
        "k",
        "ọ",
        "́",
        "▁g",
        "í",
        "ga"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 10.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁il",
        "é",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "k",
        "ọ",
        "́",
        "▁g",
        "í",
        "ga"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 13.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "il",
        "##é",
        "ẹ",
        "##̀",
        "##k",
        "##ọ́",
        "g",
        "##íg",
        "##a"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ẹ̀kọ́ gíga",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁i",
        "lé",
        "▁",
        "ẹ",
        "̀",
        "k",
        "ọ",
        "́",
        "▁",
        "gí",
        "ga"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 11.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "j",
        "ami",
        "'a"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "j",
        "ami",
        "'a"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "j",
        "ami",
        "'a"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "jami",
        "'",
        "a"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁j",
        "ami",
        "'",
        "a"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 4,
      "token_splits": [
        "jam",
        "##i",
        "'",
        "a"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "jami'a",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 3,
      "token_splits": [
        "▁jami",
        "'",
        "a"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "mah",
        "ad",
        "um"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 3,
      "token_splits": [
        "ma",
        "had",
        "um"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "mah",
        "ad",
        "um"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "ma",
        "had",
        "um"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 4,
      "token_splits": [
        "▁m",
        "ah",
        "ad",
        "um"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "maha",
        "##dum"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "mahadum",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁mahad",
        "um"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "un",
        "iversity"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "un",
        "iversity"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "un",
        "iversity"
      ],
      "english_equivalent": "university",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 92,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "university",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁university"
      ],
      "english_equivalent": "university",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "hospital",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 8,
      "token_splits": [
        "il",
        "é",
        " �",
        "�",
        "w",
        "ò",
        "s",
        "àn"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 5,
      "token_splits": [
        "ilé",
        " ì",
        "w",
        "òs",
        "àn"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 8,
      "token_splits": [
        "il",
        "Ã©",
        "ĠÃ",
        "¬",
        "w",
        "Ã²",
        "s",
        "Ãłn"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ilé",
        "▁ì",
        "w",
        "òs",
        "àn"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁il",
        "é",
        "▁",
        "ì",
        "w",
        "ò",
        "s",
        "à",
        "n"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 6,
      "token_splits": [
        "il",
        "##é",
        "ì",
        "##w",
        "##òs",
        "##àn"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "ilé ìwòsàn",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 8,
      "token_splits": [
        "▁i",
        "lé",
        "▁",
        "ì",
        "w",
        "ò",
        "s",
        "àn"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 8.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 3,
      "token_splits": [
        "as",
        "ib",
        "iti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "asib",
        "iti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 3,
      "token_splits": [
        "as",
        "ib",
        "iti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 3,
      "token_splits": [
        "as",
        "ib",
        "iti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 3,
      "token_splits": [
        "▁as",
        "ib",
        "iti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 2,
      "token_splits": [
        "asi",
        "##biti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "asibiti",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁asi",
        "biti"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 7,
      "token_splits": [
        "ụ",
        "l",
        "ọ",
        " ",
        "ọ",
        "gw",
        "ụ"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "ụlọ",
        " ọgwụ"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 7,
      "token_splits": [
        "á»¥",
        "l",
        "á»į",
        "Ġ",
        "á»į",
        "gw",
        "á»¥"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 6,
      "token_splits": [
        "ụ",
        "l",
        "ọ",
        "▁ọ",
        "gw",
        "ụ"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁",
        "ọ",
        "g",
        "w",
        "ụ"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 7,
      "token_splits": [
        "ụ",
        "##l",
        "##ọ",
        "ọ",
        "##g",
        "##w",
        "##ụ"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ụlọ ọgwụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 9,
      "token_splits": [
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁",
        "ọ",
        "g",
        "w",
        "ụ"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 93,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "hospital",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁hospital"
      ],
      "english_equivalent": "hospital",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 2,
      "token_splits": [
        "electric",
        "ity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 2,
      "token_splits": [
        "electric",
        "ity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 2,
      "token_splits": [
        "electric",
        "ity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "electricity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁electricity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "electricity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "eng",
      "language_name": "English",
      "text": "electricity",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 2,
      "token_splits": [
        "▁electric",
        "ity"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "in",
        "á",
        " m",
        "ọ",
        "̀",
        "n",
        "à",
        "m",
        "ọ",
        "́",
        "n",
        "á"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "in",
        "á",
        " mọ",
        "̀",
        "n",
        "à",
        "mọ",
        "́",
        "ná"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "inÃ¡",
        "Ġm",
        "á»į",
        "ÌĢ",
        "n",
        "Ãłm",
        "á»į",
        "Ìģ",
        "nÃ¡"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 4.5
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "in",
        "á",
        "▁m",
        "ọ",
        "̀n",
        "àm",
        "ọ",
        "́",
        "ná"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁in",
        "á",
        "▁m",
        "ọ",
        "̀",
        "n",
        "à",
        "m",
        "ọ",
        "́",
        "ná"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 11.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 9,
      "token_splits": [
        "in",
        "##á",
        "m",
        "##ọ̀",
        "##n",
        "##à",
        "##m",
        "##ọ́n",
        "##á"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 9.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "iná mọ̀nàmọ́ná",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁in",
        "á",
        "▁m",
        "ọ",
        "̀",
        "n",
        "à",
        "m",
        "ọ",
        "́",
        "ná"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 5.5
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "w",
        "utar",
        " l",
        "ant",
        "ark",
        "i"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "w",
        "utar",
        " lant",
        "arki"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "w",
        "utar",
        "Ġl",
        "ant",
        "ark",
        "i"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 4,
      "token_splits": [
        "w",
        "utar",
        "▁lant",
        "arki"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 7,
      "token_splits": [
        "▁w",
        "ut",
        "ar",
        "▁l",
        "ant",
        "ark",
        "i"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 7.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "w",
        "##utar",
        "lan",
        "##tar",
        "##ki"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "hau",
      "language_name": "Hausa",
      "text": "wutar lantarki",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁w",
        "u",
        "tar",
        "▁la",
        "ntar",
        "ki"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 6,
      "token_splits": [
        "ọ",
        "k",
        "ụ",
        " l",
        "atri",
        "k"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 4,
      "token_splits": [
        "ọ",
        "kụ",
        " lat",
        "rik"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 6,
      "token_splits": [
        "á»į",
        "k",
        "á»¥",
        "Ġl",
        "atri",
        "k"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "k",
        "ụ",
        "▁lat",
        "rik"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ọ",
        "k",
        "ụ",
        "▁lat",
        "rik"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 6.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 5,
      "token_splits": [
        "ọ",
        "##k",
        "##ụ",
        "lat",
        "##rik"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "ọkụ latrik",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 6,
      "token_splits": [
        "▁",
        "ọ",
        "k",
        "ụ",
        "▁la",
        "trik"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 1,
      "token_splits": [
        "light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 1,
      "token_splits": [
        "light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 1,
      "token_splits": [
        "light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 1,
      "token_splits": [
        "light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 1,
      "token_splits": [
        "▁light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 1,
      "token_splits": [
        "light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 1,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 94,
      "category": "compound_words",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "light",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 1,
      "token_splits": [
        "▁light"
      ],
      "english_equivalent": "electricity",
      "english_tokens": 2,
      "token_ratio_vs_english": 0.5
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Traffic",
        " in",
        " Lagos",
        " is",
        " terrible",
        " during",
        " rush",
        " hour",
        ",",
        " everyone",
        " is",
        " stuck",
        " on",
        " the",
        " road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Traffic",
        " in",
        " Lagos",
        " is",
        " terrible",
        " during",
        " rush",
        " hour",
        ",",
        " everyone",
        " is",
        " stuck",
        " on",
        " the",
        " road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "Traffic",
        "Ġin",
        "ĠLagos",
        "Ġis",
        "Ġterrible",
        "Ġduring",
        "Ġrush",
        "Ġhour",
        ",",
        "Ġeveryone",
        "Ġis",
        "Ġstuck",
        "Ġon",
        "Ġthe",
        "Ġroad"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 15,
      "token_splits": [
        "Traffic",
        "▁in",
        "▁Lagos",
        "▁is",
        "▁terrible",
        "▁during",
        "▁rush",
        "▁hour",
        ",",
        "▁everyone",
        "▁is",
        "▁stuck",
        "▁on",
        "▁the",
        "▁road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 17,
      "token_splits": [
        "▁Tra",
        "ffic",
        "▁in",
        "▁Lag",
        "os",
        "▁is",
        "▁terrible",
        "▁during",
        "▁rush",
        "▁hour",
        ",",
        "▁everyone",
        "▁is",
        "▁stuck",
        "▁on",
        "▁the",
        "▁road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 17,
      "token_splits": [
        "Traffic",
        "in",
        "Lagos",
        "is",
        "terrible",
        "during",
        "rus",
        "##h",
        "hour",
        ",",
        "everyone",
        "is",
        "st",
        "##uck",
        "on",
        "the",
        "road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 17,
      "token_splits": [
        "▁Traffic",
        "▁in",
        "▁Lago",
        "s",
        "▁is",
        "▁terrible",
        "▁during",
        "▁",
        "rush",
        "▁hour",
        ",",
        "▁everyone",
        "▁is",
        "▁stuck",
        "▁on",
        "▁the",
        "▁road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 62,
      "token_splits": [
        "�",
        "�",
        "k",
        "ọ",
        "̀",
        " p",
        "ọ",
        "̀",
        " gan",
        " n",
        "í",
        " �",
        "�",
        "k",
        "ó",
        " n",
        "í",
        " à",
        "k",
        "ó",
        "k",
        "ò",
        " tí",
        " gb",
        "og",
        "bo",
        " è",
        "n",
        "ì",
        "y",
        "àn",
        " ",
        "ń",
        " l",
        "ọ",
        " sí",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́",
        ",",
        " gb",
        "og",
        "bo",
        " è",
        "n",
        "ì",
        "y",
        "àn",
        " d",
        "ú",
        "ró",
        " l",
        "ój",
        "ú",
        " ",
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 4.133
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 41,
      "token_splits": [
        "Ọ",
        "kọ",
        "̀",
        " pọ",
        "̀",
        " gan",
        " ní",
        " È",
        "k",
        "ó",
        " ní",
        " à",
        "k",
        "ók",
        "ò",
        " tí",
        " gbogbo",
        " èn",
        "ì",
        "y",
        "àn",
        " ń",
        " lọ",
        " sí",
        "ṣẹ",
        "́",
        ",",
        " gbogbo",
        " èn",
        "ì",
        "y",
        "àn",
        " dú",
        "ró",
        " l",
        "ój",
        "ú",
        " ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.733
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 58,
      "token_splits": [
        "á»",
        "Į",
        "k",
        "á»į",
        "ÌĢ",
        "Ġp",
        "á»į",
        "ÌĢ",
        "Ġgan",
        "ĠnÃŃ",
        "ĠÃ",
        "Ī",
        "k",
        "Ã³",
        "ĠnÃŃ",
        "ĠÃł",
        "k",
        "Ã³",
        "k",
        "Ã²",
        "ĠtÃŃ",
        "Ġgb",
        "og",
        "bo",
        "ĠÃ¨",
        "n",
        "Ã¬",
        "y",
        "Ãłn",
        "Ġ",
        "ÅĦ",
        "Ġl",
        "á»į",
        "ĠsÃŃ",
        "á¹",
        "£",
        "áº¹",
        "Ìģ",
        ",",
        "Ġgb",
        "og",
        "bo",
        "ĠÃ¨",
        "n",
        "Ã¬",
        "y",
        "Ãłn",
        "Ġd",
        "Ãº",
        "rÃ³",
        "Ġl",
        "Ã³j",
        "Ãº",
        "Ġ",
        "á»į",
        "ÌĢ",
        "n",
        "Ãł"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.867
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 49,
      "token_splits": [
        "Ọ",
        "k",
        "ọ",
        "̀",
        "▁p",
        "ọ",
        "̀",
        "▁gan",
        "▁ní",
        "▁È",
        "kó",
        "▁ní",
        "▁à",
        "k",
        "ók",
        "ò",
        "▁tí",
        "▁gb",
        "og",
        "bo",
        "▁è",
        "n",
        "ì",
        "y",
        "àn",
        "▁",
        "ń",
        "▁lọ",
        "▁sí",
        "ṣ",
        "ẹ",
        "́",
        ",",
        "▁gb",
        "og",
        "bo",
        "▁è",
        "n",
        "ì",
        "y",
        "àn",
        "▁dú",
        "ró",
        "▁l",
        "ój",
        "ú",
        "▁ọ",
        "̀n",
        "à"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.267
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 67,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "k",
        "ọ",
        "̀",
        "▁p",
        "ọ",
        "̀",
        "▁gan",
        "▁n",
        "í",
        "▁È",
        "k",
        "ó",
        "▁n",
        "í",
        "▁à",
        "k",
        "ó",
        "k",
        "ò",
        "▁t",
        "í",
        "▁g",
        "b",
        "og",
        "bo",
        "▁è",
        "n",
        "ì",
        "y",
        "à",
        "n",
        "▁",
        "ń",
        "▁l",
        "ọ",
        "▁sí",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        ",",
        "▁g",
        "b",
        "og",
        "bo",
        "▁è",
        "n",
        "ì",
        "y",
        "à",
        "n",
        "▁d",
        "ú",
        "ró",
        "▁l",
        "ój",
        "ú",
        "▁",
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 3.941
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 48,
      "token_splits": [
        "Ọ",
        "##k",
        "##ọ̀",
        "p",
        "##ọ̀",
        "gan",
        "ní",
        "È",
        "##kó",
        "ní",
        "à",
        "##kó",
        "##k",
        "##ò",
        "tí",
        "g",
        "##bo",
        "##gbo",
        "è",
        "##n",
        "##ì",
        "##y",
        "##àn",
        "ń",
        "l",
        "##ọ",
        "sí",
        "##ṣ",
        "##ẹ́",
        ",",
        "g",
        "##bo",
        "##gbo",
        "è",
        "##n",
        "##ì",
        "##y",
        "##àn",
        "d",
        "##úr",
        "##ó",
        "l",
        "##ój",
        "##ú",
        "ọ",
        "##̀",
        "##n",
        "##à"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 2.824
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ọkọ̀ pọ̀ gan ní Èkó ní àkókò tí gbogbo ènìyàn ń lọ síṣẹ́, gbogbo ènìyàn dúró lójú ọ̀nà",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 52,
      "token_splits": [
        "▁",
        "Ọ",
        "k",
        "ọ",
        "̀",
        "▁p",
        "ọ",
        "̀",
        "▁gan",
        "▁ní",
        "▁È",
        "kó",
        "▁ní",
        "▁à",
        "k",
        "ók",
        "ò",
        "▁tí",
        "▁g",
        "bog",
        "bo",
        "▁è",
        "n",
        "ì",
        "y",
        "àn",
        "▁",
        "ń",
        "▁l",
        "ọ",
        "▁sí",
        "ṣ",
        "ẹ",
        "́",
        ",",
        "▁g",
        "bog",
        "bo",
        "▁è",
        "n",
        "ì",
        "y",
        "àn",
        "▁dú",
        "ró",
        "▁ló",
        "jú",
        "▁",
        "ọ",
        "̀",
        "n",
        "à"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 3.059
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 29,
      "token_splits": [
        "C",
        "unk",
        "os",
        "on",
        " ab",
        "aben",
        " mot",
        "a",
        " a",
        " Leg",
        "as",
        " y",
        "ana",
        " da",
        " m",
        "uni",
        " lok",
        "acin",
        " t",
        "ashi",
        ",",
        " k",
        "owa",
        " y",
        "ana",
        " mak",
        "ale",
        " a",
        " hanya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.933
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 24,
      "token_splits": [
        "C",
        "unk",
        "os",
        "on",
        " ab",
        "aben",
        " mota",
        " a",
        " Leg",
        "as",
        " yana",
        " da",
        " muni",
        " lokacin",
        " t",
        "ashi",
        ",",
        " k",
        "owa",
        " yana",
        " mak",
        "ale",
        " a",
        " hanya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 27,
      "token_splits": [
        "C",
        "unk",
        "os",
        "on",
        "Ġab",
        "aben",
        "Ġmot",
        "a",
        "Ġa",
        "ĠLeg",
        "as",
        "Ġyana",
        "Ġda",
        "Ġm",
        "uni",
        "Ġlok",
        "acin",
        "Ġt",
        "ashi",
        ",",
        "Ġk",
        "owa",
        "Ġyana",
        "Ġmak",
        "ale",
        "Ġa",
        "Ġhanya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.8
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 26,
      "token_splits": [
        "C",
        "unk",
        "os",
        "on",
        "▁ab",
        "aben",
        "▁mo",
        "ta",
        "▁a",
        "▁Le",
        "gas",
        "▁yana",
        "▁da",
        "▁muni",
        "▁lok",
        "acin",
        "▁t",
        "ashi",
        ",",
        "▁k",
        "owa",
        "▁yana",
        "▁mak",
        "ale",
        "▁a",
        "▁hanya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.733
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 33,
      "token_splits": [
        "▁C",
        "unk",
        "os",
        "on",
        "▁ab",
        "aben",
        "▁mot",
        "a",
        "▁a",
        "▁Leg",
        "as",
        "▁y",
        "ana",
        "▁da",
        "▁m",
        "uni",
        "▁l",
        "ok",
        "ac",
        "in",
        "▁t",
        "ashi",
        ",",
        "▁k",
        "owa",
        "▁y",
        "ana",
        "▁m",
        "ak",
        "ale",
        "▁a",
        "▁h",
        "anya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.941
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 26,
      "token_splits": [
        "Cu",
        "##nko",
        "##son",
        "ab",
        "##aben",
        "mota",
        "a",
        "Lega",
        "##s",
        "yana",
        "da",
        "mun",
        "##i",
        "lo",
        "##ka",
        "##cin",
        "tas",
        "##hi",
        ",",
        "ko",
        "##wa",
        "yana",
        "maka",
        "##le",
        "a",
        "hanya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.529
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Cunkoson ababen mota a Legas yana da muni lokacin tashi, kowa yana makale a hanya",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 23,
      "token_splits": [
        "▁Cu",
        "nko",
        "son",
        "▁aba",
        "ben",
        "▁mota",
        "▁a",
        "▁Lega",
        "s",
        "▁yana",
        "▁da",
        "▁muni",
        "▁lokacin",
        "▁tash",
        "i",
        ",",
        "▁",
        "kowa",
        "▁yana",
        "▁maka",
        "le",
        "▁a",
        "▁hanya"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 1.353
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 46,
      "token_splits": [
        "�",
        "�",
        "k",
        "ụ",
        "k",
        "ọ",
        " ",
        "ụ",
        "gb",
        "ọ",
        " al",
        "a",
        " na",
        " Lagos",
        " j",
        "ọ",
        "r",
        "ọ",
        " nj",
        "ọ",
        " n",
        "'",
        "oge",
        " ",
        "ọ",
        "r",
        "ụ",
        ",",
        " on",
        "ye",
        " ",
        "ọ",
        " b",
        "ụ",
        "la",
        " n",
        "ọ",
        " n",
        "'",
        "ok",
        "por",
        "o",
        " ",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.067
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 23,
      "token_splits": [
        "Ọ",
        "kụ",
        "kọ",
        " ụgbọ",
        " ala",
        " na",
        " Lagos",
        " j",
        "ọrọ",
        " nj",
        "ọ",
        " n",
        "'oge",
        " ọrụ",
        ",",
        " onye",
        " ọ",
        " bụla",
        " nọ",
        " n",
        "'ok",
        "poro",
        " ụzọ"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.533
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 45,
      "token_splits": [
        "á»",
        "Į",
        "k",
        "á»¥",
        "k",
        "á»į",
        "Ġ",
        "á»¥",
        "gb",
        "á»į",
        "Ġal",
        "a",
        "Ġna",
        "ĠLagos",
        "Ġj",
        "á»į",
        "r",
        "á»į",
        "Ġnj",
        "á»į",
        "Ġn",
        "'",
        "oge",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        ",",
        "Ġon",
        "ye",
        "Ġ",
        "á»į",
        "Ġbá»¥",
        "la",
        "Ġn",
        "á»į",
        "Ġn",
        "'",
        "ok",
        "por",
        "o",
        "Ġ",
        "á»¥",
        "z",
        "á»į"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 40,
      "token_splits": [
        "Ọ",
        "k",
        "ụ",
        "k",
        "ọ",
        "▁",
        "ụ",
        "gb",
        "ọ",
        "▁ala",
        "▁na",
        "▁Lagos",
        "▁j",
        "ọ",
        "r",
        "ọ",
        "▁nj",
        "ọ",
        "▁n",
        "'",
        "oge",
        "▁ọ",
        "r",
        "ụ",
        ",",
        "▁on",
        "ye",
        "▁ọ",
        "▁bụ",
        "la",
        "▁n",
        "ọ",
        "▁n",
        "'",
        "ok",
        "poro",
        "▁",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.667
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 49,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "k",
        "ụ",
        "k",
        "ọ",
        "▁",
        "ụ",
        "gb",
        "ọ",
        "▁al",
        "a",
        "▁na",
        "▁Lag",
        "os",
        "▁j",
        "ọ",
        "r",
        "ọ",
        "▁nj",
        "ọ",
        "▁n",
        "'",
        "oge",
        "▁",
        "ọ",
        "r",
        "ụ",
        ",",
        "▁on",
        "ye",
        "▁",
        "ọ",
        "▁b",
        "ụ",
        "la",
        "▁n",
        "ọ",
        "▁n",
        "'",
        "ok",
        "por",
        "o",
        "▁",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 2.882
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 43,
      "token_splits": [
        "Ọ",
        "##k",
        "##ụ",
        "##k",
        "##ọ",
        "ụ",
        "##g",
        "##b",
        "##ọ",
        "ala",
        "na",
        "Lagos",
        "j",
        "##ọ",
        "##r",
        "##ọ",
        "n",
        "##j",
        "##ọ",
        "n",
        "'",
        "og",
        "##e",
        "ọ",
        "##r",
        "##ụ",
        ",",
        "on",
        "##ye",
        "ọ",
        "b",
        "##ụ",
        "##la",
        "n",
        "##ọ",
        "n",
        "'",
        "ok",
        "##por",
        "##o",
        "ụ",
        "##z",
        "##ọ"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 2.529
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụkọ ụgbọ ala na Lagos jọrọ njọ n'oge ọrụ, onye ọ bụla nọ n'okporo ụzọ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 47,
      "token_splits": [
        "▁",
        "Ọ",
        "k",
        "ụ",
        "k",
        "ọ",
        "▁",
        "ụ",
        "gb",
        "ọ",
        "▁ala",
        "▁na",
        "▁Lago",
        "s",
        "▁j",
        "ọ",
        "r",
        "ọ",
        "▁n",
        "j",
        "ọ",
        "▁n",
        "'",
        "o",
        "ge",
        "▁",
        "ọ",
        "r",
        "ụ",
        ",",
        "▁on",
        "ye",
        "▁",
        "ọ",
        "▁b",
        "ụ",
        "la",
        "▁n",
        "ọ",
        "▁n",
        "'",
        "ok",
        "poro",
        "▁",
        "ụ",
        "z",
        "ọ"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 2.765
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 14,
      "token_splits": [
        "L",
        "agos",
        " traffic",
        " bad",
        " die",
        " during",
        " rush",
        " hour",
        ",",
        " everybody",
        " don",
        " stuck",
        " for",
        " road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 0.933
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 14,
      "token_splits": [
        "L",
        "agos",
        " traffic",
        " bad",
        " die",
        " during",
        " rush",
        " hour",
        ",",
        " everybody",
        " don",
        " stuck",
        " for",
        " road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 0.933
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 14,
      "token_splits": [
        "L",
        "agos",
        "Ġtraffic",
        "Ġbad",
        "Ġdie",
        "Ġduring",
        "Ġrush",
        "Ġhour",
        ",",
        "Ġeverybody",
        "Ġdon",
        "Ġstuck",
        "Ġfor",
        "Ġroad"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 0.933
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "Lagos",
        "▁traffic",
        "▁bad",
        "▁die",
        "▁during",
        "▁rush",
        "▁hour",
        ",",
        "▁everybody",
        "▁don",
        "▁stuck",
        "▁for",
        "▁road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 15,
      "token_ratio_vs_english": 0.867
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁Lag",
        "os",
        "▁traffic",
        "▁bad",
        "▁die",
        "▁during",
        "▁rush",
        "▁hour",
        ",",
        "▁everybody",
        "▁don",
        "▁stuck",
        "▁for",
        "▁road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 0.824
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 16,
      "token_splits": [
        "Lagos",
        "traffic",
        "bad",
        "die",
        "during",
        "rus",
        "##h",
        "hour",
        ",",
        "every",
        "##body",
        "don",
        "st",
        "##uck",
        "for",
        "road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 0.941
    },
    {
      "entry_id": 95,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Lagos traffic bad die during rush hour, everybody don stuck for road",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁Lago",
        "s",
        "▁traffic",
        "▁bad",
        "▁die",
        "▁during",
        "▁",
        "rush",
        "▁hour",
        ",",
        "▁everybody",
        "▁don",
        "▁stuck",
        "▁for",
        "▁road"
      ],
      "english_equivalent": "Traffic in Lagos is terrible during rush hour, everyone is stuck on the road",
      "english_tokens": 17,
      "token_ratio_vs_english": 0.882
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 13,
      "token_splits": [
        "The",
        " electricity",
        " has",
        " been",
        " off",
        " for",
        " three",
        " days",
        ",",
        " we",
        " are",
        " using",
        " generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 13,
      "token_splits": [
        "The",
        " electricity",
        " has",
        " been",
        " off",
        " for",
        " three",
        " days",
        ",",
        " we",
        " are",
        " using",
        " generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 13,
      "token_splits": [
        "The",
        "Ġelectricity",
        "Ġhas",
        "Ġbeen",
        "Ġoff",
        "Ġfor",
        "Ġthree",
        "Ġdays",
        ",",
        "Ġwe",
        "Ġare",
        "Ġusing",
        "Ġgenerator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 13,
      "token_splits": [
        "The",
        "▁electricity",
        "▁has",
        "▁been",
        "▁off",
        "▁for",
        "▁three",
        "▁days",
        ",",
        "▁we",
        "▁are",
        "▁using",
        "▁generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 13,
      "token_splits": [
        "▁The",
        "▁electricity",
        "▁has",
        "▁been",
        "▁off",
        "▁for",
        "▁three",
        "▁days",
        ",",
        "▁we",
        "▁are",
        "▁using",
        "▁generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 13,
      "token_splits": [
        "The",
        "electricity",
        "has",
        "been",
        "off",
        "for",
        "three",
        "days",
        ",",
        "we",
        "are",
        "using",
        "generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The electricity has been off for three days, we are using generator",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁The",
        "▁electric",
        "ity",
        "▁has",
        "▁been",
        "▁off",
        "▁for",
        "▁three",
        "▁days",
        ",",
        "▁we",
        "▁are",
        "▁using",
        "▁generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 46,
      "token_splits": [
        "In",
        "á",
        " m",
        "ọ",
        "̀",
        "n",
        "à",
        "m",
        "ọ",
        "́",
        "n",
        "á",
        " ti",
        " p",
        "à",
        "d",
        "án",
        "ù",
        " f",
        "ún",
        " ",
        "ọ",
        "j",
        "ọ",
        "́",
        " m",
        "�",
        "�",
        "́",
        "ta",
        ",",
        " a",
        " ",
        "ń",
        " lo",
        " ",
        "�",
        "�",
        "̀",
        "r",
        "ọ",
        " am",
        "ún",
        "á",
        "w",
        "á"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 3.538
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 32,
      "token_splits": [
        "In",
        "á",
        " mọ",
        "̀",
        "n",
        "à",
        "mọ",
        "́",
        "ná",
        " ti",
        " pà",
        "d",
        "án",
        "ù",
        " fún",
        " ọjọ",
        "́",
        " mẹ",
        "́",
        "ta",
        ",",
        " a",
        " ń",
        " lo",
        " ẹ",
        "̀",
        "rọ",
        " am",
        "ún",
        "á",
        "w",
        "á"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.462
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 41,
      "token_splits": [
        "In",
        "Ã¡",
        "Ġm",
        "á»į",
        "ÌĢ",
        "n",
        "Ãłm",
        "á»į",
        "Ìģ",
        "nÃ¡",
        "Ġti",
        "Ġp",
        "Ãł",
        "d",
        "Ã¡n",
        "Ã¹",
        "Ġf",
        "Ãºn",
        "Ġ",
        "á»į",
        "j",
        "á»į",
        "Ìģ",
        "Ġmáº¹",
        "Ìģ",
        "ta",
        ",",
        "Ġa",
        "Ġ",
        "ÅĦ",
        "Ġlo",
        "Ġ",
        "áº¹",
        "ÌĢ",
        "r",
        "á»į",
        "Ġam",
        "Ãºn",
        "Ã¡",
        "w",
        "Ã¡"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 3.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 38,
      "token_splits": [
        "In",
        "á",
        "▁m",
        "ọ",
        "̀n",
        "àm",
        "ọ",
        "́",
        "ná",
        "▁ti",
        "▁pà",
        "d",
        "án",
        "ù",
        "▁f",
        "ún",
        "▁ọ",
        "j",
        "ọ",
        "́",
        "▁mẹ",
        "́",
        "ta",
        ",",
        "▁a",
        "▁",
        "ń",
        "▁lo",
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁am",
        "ún",
        "á",
        "w",
        "á"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.923
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 47,
      "token_splits": [
        "▁In",
        "á",
        "▁m",
        "ọ",
        "̀",
        "n",
        "à",
        "m",
        "ọ",
        "́",
        "ná",
        "▁ti",
        "▁p",
        "à",
        "d",
        "án",
        "ù",
        "▁f",
        "ún",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁m",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "ta",
        ",",
        "▁a",
        "▁",
        "ń",
        "▁lo",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ọ",
        "▁am",
        "ún",
        "á",
        "w",
        "á"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 3.615
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 36,
      "token_splits": [
        "In",
        "##á",
        "m",
        "##ọ̀",
        "##n",
        "##à",
        "##m",
        "##ọ́n",
        "##á",
        "ti",
        "p",
        "##à",
        "##dá",
        "##n",
        "##ù",
        "f",
        "##ún",
        "ọ",
        "##j",
        "##ọ́",
        "mẹ",
        "##́",
        "##ta",
        ",",
        "a",
        "ń",
        "lo",
        "ẹ",
        "##̀",
        "##r",
        "##ọ",
        "am",
        "##ún",
        "##á",
        "##w",
        "##á"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.769
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Iná mọ̀nàmọ́ná ti pàdánù fún ọjọ́ mẹ́ta, a ń lo ẹ̀rọ amúnáwá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 40,
      "token_splits": [
        "▁I",
        "ná",
        "▁m",
        "ọ",
        "̀",
        "n",
        "à",
        "m",
        "ọ",
        "́",
        "ná",
        "▁ti",
        "▁pà",
        "dán",
        "ù",
        "▁fú",
        "n",
        "▁",
        "ọ",
        "j",
        "ọ",
        "́",
        "▁mẹ",
        "́",
        "ta",
        ",",
        "▁a",
        "▁",
        "ń",
        "▁lo",
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁am",
        "ú",
        "ná",
        "w",
        "á"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.857
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 23,
      "token_splits": [
        "W",
        "utar",
        " l",
        "ant",
        "ark",
        "i",
        " ta",
        " d",
        "aina",
        " kw",
        "ana",
        " u",
        "ku",
        ",",
        " m",
        "una",
        " am",
        "f",
        "ani",
        " da",
        " jan",
        "are",
        "ta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.769
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "W",
        "utar",
        " lant",
        "arki",
        " ta",
        " d",
        "aina",
        " kw",
        "ana",
        " uku",
        ",",
        " muna",
        " amfani",
        " da",
        " jan",
        "are",
        "ta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.308
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 23,
      "token_splits": [
        "W",
        "utar",
        "Ġl",
        "ant",
        "ark",
        "i",
        "Ġta",
        "Ġd",
        "aina",
        "Ġkw",
        "ana",
        "Ġu",
        "ku",
        ",",
        "Ġm",
        "una",
        "Ġam",
        "f",
        "ani",
        "Ġda",
        "Ġjan",
        "are",
        "ta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.769
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "W",
        "utar",
        "▁lant",
        "arki",
        "▁ta",
        "▁d",
        "aina",
        "▁kw",
        "ana",
        "▁uku",
        ",",
        "▁muna",
        "▁am",
        "fani",
        "▁da",
        "▁jan",
        "are",
        "ta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.385
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 25,
      "token_splits": [
        "▁W",
        "ut",
        "ar",
        "▁l",
        "ant",
        "ark",
        "i",
        "▁ta",
        "▁d",
        "ain",
        "a",
        "▁kw",
        "ana",
        "▁u",
        "ku",
        ",",
        "▁m",
        "una",
        "▁am",
        "f",
        "ani",
        "▁da",
        "▁jan",
        "are",
        "ta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.923
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 21,
      "token_splits": [
        "Wu",
        "##tar",
        "lan",
        "##tar",
        "##ki",
        "ta",
        "dai",
        "##na",
        "kwa",
        "##na",
        "uk",
        "##u",
        ",",
        "mun",
        "##a",
        "am",
        "##fan",
        "##i",
        "da",
        "janar",
        "##eta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.615
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Wutar lantarki ta daina kwana uku, muna amfani da janareta",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 17,
      "token_splits": [
        "▁Wu",
        "tar",
        "▁la",
        "ntar",
        "ki",
        "▁ta",
        "▁da",
        "ina",
        "▁kwa",
        "na",
        "▁uku",
        ",",
        "▁muna",
        "▁amfani",
        "▁da",
        "▁jana",
        "reta"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.214
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 28,
      "token_splits": [
        "�",
        "�",
        "k",
        "ụ",
        " l",
        "atri",
        "k",
        " ap",
        "ụ",
        "ọ",
        "la",
        " ",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        " at",
        "ọ",
        ",",
        " any",
        "ị",
        " na",
        "-e",
        "ji",
        " j",
        "ener",
        "eto"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Ọ",
        "kụ",
        " lat",
        "rik",
        " ap",
        "ụọ",
        "la",
        " ụbọchị",
        " atọ",
        ",",
        " anyị",
        " na",
        "-eji",
        " jener",
        "eto"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 28,
      "token_splits": [
        "á»",
        "Į",
        "k",
        "á»¥",
        "Ġl",
        "atri",
        "k",
        "Ġap",
        "á»¥",
        "á»į",
        "la",
        "Ġ",
        "á»¥",
        "b",
        "á»į",
        "ch",
        "á»ĭ",
        "Ġat",
        "á»į",
        ",",
        "Ġany",
        "á»ĭ",
        "Ġna",
        "-e",
        "ji",
        "Ġj",
        "ener",
        "eto"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 27,
      "token_splits": [
        "Ọ",
        "k",
        "ụ",
        "▁lat",
        "rik",
        "▁ap",
        "ụ",
        "ọ",
        "la",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁at",
        "ọ",
        ",",
        "▁any",
        "ị",
        "▁na",
        "-",
        "e",
        "ji",
        "▁jen",
        "ere",
        "to"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.077
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 30,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "k",
        "ụ",
        "▁lat",
        "rik",
        "▁ap",
        "ụ",
        "ọ",
        "la",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁at",
        "ọ",
        ",",
        "▁any",
        "ị",
        "▁na",
        "-",
        "e",
        "ji",
        "▁j",
        "ener",
        "eto"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 2.308
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 24,
      "token_splits": [
        "Ọ",
        "##k",
        "##ụ",
        "lat",
        "##rik",
        "ap",
        "##ụ",
        "##ọ",
        "##la",
        "ụ",
        "##b",
        "##ọc",
        "##hị",
        "at",
        "##ọ",
        ",",
        "any",
        "##ị",
        "na",
        "-",
        "ej",
        "##i",
        "jener",
        "##eto"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.846
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọkụ latrik apụọla ụbọchị atọ, anyị na-eji jenereto",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 28,
      "token_splits": [
        "▁",
        "Ọ",
        "k",
        "ụ",
        "▁la",
        "trik",
        "▁ap",
        "ụ",
        "ọ",
        "la",
        "▁",
        "ụ",
        "b",
        "ọ",
        "ch",
        "ị",
        "▁at",
        "ọ",
        ",",
        "▁any",
        "ị",
        "▁na",
        "-",
        "e",
        "ji",
        "▁je",
        "ne",
        "reto"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "token_splits": [
        "NE",
        "PA",
        " don",
        " take",
        " light",
        " for",
        " three",
        " days",
        " now",
        ",",
        " we",
        " de",
        "y",
        " use",
        " generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 14,
      "token_splits": [
        "NE",
        "PA",
        " don",
        " take",
        " light",
        " for",
        " three",
        " days",
        " now",
        ",",
        " we",
        " dey",
        " use",
        " generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.077
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 15,
      "token_splits": [
        "NE",
        "PA",
        "Ġdon",
        "Ġtake",
        "Ġlight",
        "Ġfor",
        "Ġthree",
        "Ġdays",
        "Ġnow",
        ",",
        "Ġwe",
        "Ġde",
        "y",
        "Ġuse",
        "Ġgenerator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 14,
      "token_splits": [
        "NE",
        "PA",
        "▁don",
        "▁take",
        "▁light",
        "▁for",
        "▁three",
        "▁days",
        "▁now",
        ",",
        "▁we",
        "▁dey",
        "▁use",
        "▁generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.077
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 15,
      "token_splits": [
        "▁NE",
        "PA",
        "▁don",
        "▁take",
        "▁light",
        "▁for",
        "▁three",
        "▁days",
        "▁now",
        ",",
        "▁we",
        "▁de",
        "y",
        "▁use",
        "▁generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "NE",
        "##PA",
        "don",
        "take",
        "light",
        "for",
        "three",
        "days",
        "now",
        ",",
        "we",
        "de",
        "##y",
        "use",
        "generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 13,
      "token_ratio_vs_english": 1.154
    },
    {
      "entry_id": 96,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "NEPA don take light for three days now, we dey use generator",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁N",
        "EPA",
        "▁don",
        "▁take",
        "▁light",
        "▁for",
        "▁three",
        "▁days",
        "▁now",
        ",",
        "▁we",
        "▁de",
        "y",
        "▁use",
        "▁generator"
      ],
      "english_equivalent": "The electricity has been off for three days, we are using generator",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.071
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 12,
      "token_splits": [
        "My",
        " grandmother",
        " makes",
        " the",
        " best",
        " eg",
        "usi",
        " soup",
        " with",
        " pounded",
        " y",
        "am"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 12,
      "token_splits": [
        "My",
        " grandmother",
        " makes",
        " the",
        " best",
        " eg",
        "usi",
        " soup",
        " with",
        " p",
        "ounded",
        " yam"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 12,
      "token_splits": [
        "My",
        "Ġgrandmother",
        "Ġmakes",
        "Ġthe",
        "Ġbest",
        "Ġeg",
        "usi",
        "Ġsoup",
        "Ġwith",
        "Ġpounded",
        "Ġy",
        "am"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 12,
      "token_splits": [
        "My",
        "▁grandmother",
        "▁makes",
        "▁the",
        "▁best",
        "▁e",
        "gu",
        "si",
        "▁soup",
        "▁with",
        "▁pounded",
        "▁yam"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁My",
        "▁grandmother",
        "▁makes",
        "▁the",
        "▁best",
        "▁eg",
        "us",
        "i",
        "▁soup",
        "▁with",
        "▁pound",
        "ed",
        "▁y",
        "am"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 15,
      "token_splits": [
        "My",
        "grandmother",
        "makes",
        "the",
        "best",
        "e",
        "##gus",
        "##i",
        "sou",
        "##p",
        "with",
        "pound",
        "##ed",
        "ya",
        "##m"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "My grandmother makes the best egusi soup with pounded yam",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁My",
        "▁grand",
        "mo",
        "ther",
        "▁makes",
        "▁the",
        "▁best",
        "▁e",
        "gusi",
        "▁soup",
        "▁with",
        "▁p",
        "ounded",
        "▁ya",
        "m"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 45,
      "token_splits": [
        "�",
        "�",
        "y",
        "á",
        " à",
        "gb",
        "à",
        " mi",
        " l",
        "ó",
        " ",
        "ń",
        " �",
        "�",
        "�",
        "e",
        " ",
        "ọ",
        "b",
        "�",
        "�",
        "̀",
        " ",
        "�",
        "�",
        "g",
        "ús",
        "í",
        " t",
        "ó",
        " d",
        "á",
        "ra",
        " j",
        "ù",
        " l",
        "ọ",
        " p",
        "�",
        "�",
        "̀",
        "l",
        "ú",
        " iy",
        "án"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.75
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 31,
      "token_splits": [
        "Ì",
        "y",
        "á",
        " à",
        "gb",
        "à",
        " mi",
        " ló",
        " ń",
        " ṣe",
        " ọ",
        "b",
        "ẹ",
        "̀",
        " ẹ",
        "g",
        "ús",
        "í",
        " tó",
        " dá",
        "ra",
        " j",
        "ù",
        " lọ",
        " p",
        "ẹ",
        "̀",
        "l",
        "ú",
        " iy",
        "án"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.583
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 40,
      "token_splits": [
        "ÃĮ",
        "y",
        "Ã¡",
        "ĠÃł",
        "gb",
        "Ãł",
        "Ġmi",
        "Ġl",
        "Ã³",
        "Ġ",
        "ÅĦ",
        "Ġá",
        "¹",
        "£",
        "e",
        "Ġ",
        "á»į",
        "b",
        "áº¹",
        "ÌĢ",
        "Ġ",
        "áº¹",
        "g",
        "Ãºs",
        "ÃŃ",
        "Ġt",
        "Ã³",
        "ĠdÃ¡",
        "ra",
        "Ġj",
        "Ã¹",
        "Ġl",
        "á»į",
        "Ġp",
        "áº¹",
        "ÌĢ",
        "l",
        "Ãº",
        "Ġiy",
        "Ã¡n"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 35,
      "token_splits": [
        "Ì",
        "y",
        "á",
        "▁à",
        "gb",
        "à",
        "▁mi",
        "▁ló",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁ọ",
        "b",
        "ẹ",
        "̀",
        "▁",
        "ẹ",
        "g",
        "ús",
        "í",
        "▁tó",
        "▁dá",
        "ra",
        "▁j",
        "ù",
        "▁lọ",
        "▁p",
        "ẹ",
        "̀",
        "l",
        "ú",
        "▁iy",
        "án"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.917
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 47,
      "token_splits": [
        "▁",
        "Ì",
        "y",
        "á",
        "▁à",
        "gb",
        "à",
        "▁mi",
        "▁l",
        "ó",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁",
        "ọ",
        "b",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "g",
        "ús",
        "í",
        "▁t",
        "ó",
        "▁d",
        "ára",
        "▁j",
        "ù",
        "▁l",
        "ọ",
        "▁p",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "l",
        "ú",
        "▁i",
        "y",
        "án"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 14,
      "token_ratio_vs_english": 3.357
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 32,
      "token_splits": [
        "Ì",
        "##y",
        "##á",
        "à",
        "##g",
        "##bà",
        "mi",
        "l",
        "##ó",
        "ń",
        "ṣ",
        "##e",
        "ọ",
        "##b",
        "##ẹ̀",
        "ẹ",
        "##g",
        "##ús",
        "##í",
        "tó",
        "dá",
        "##ra",
        "j",
        "##ù",
        "l",
        "##ọ",
        "p",
        "##ẹ̀",
        "##lú",
        "i",
        "##y",
        "##án"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.133
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ìyá àgbà mi ló ń ṣe ọbẹ̀ ẹgúsí tó dára jù lọ pẹ̀lú iyán",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 38,
      "token_splits": [
        "▁",
        "Ì",
        "y",
        "á",
        "▁à",
        "gb",
        "à",
        "▁mi",
        "▁ló",
        "▁",
        "ń",
        "▁",
        "ṣ",
        "e",
        "▁",
        "ọ",
        "b",
        "ẹ",
        "̀",
        "▁",
        "ẹ",
        "g",
        "ús",
        "í",
        "▁tó",
        "▁d",
        "ára",
        "▁j",
        "ù",
        "▁l",
        "ọ",
        "▁p",
        "ẹ",
        "̀",
        "lú",
        "▁i",
        "y",
        "án"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.533
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 19,
      "token_splits": [
        "K",
        "akk",
        "ata",
        " t",
        "afi",
        " k",
        "owa",
        " w",
        "aj",
        "en",
        " d",
        "afa",
        " mi",
        "yan",
        " gy",
        "ada",
        " da",
        " tu",
        "wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.583
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 17,
      "token_splits": [
        "K",
        "akk",
        "ata",
        " t",
        "afi",
        " k",
        "owa",
        " wajen",
        " da",
        "fa",
        " m",
        "iyan",
        " gy",
        "ada",
        " da",
        " tu",
        "wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.417
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 19,
      "token_splits": [
        "K",
        "akk",
        "ata",
        "Ġt",
        "afi",
        "Ġk",
        "owa",
        "Ġw",
        "aj",
        "en",
        "Ġd",
        "afa",
        "Ġmi",
        "yan",
        "Ġgy",
        "ada",
        "Ġda",
        "Ġtu",
        "wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.583
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 18,
      "token_splits": [
        "Kak",
        "kata",
        "▁ta",
        "fi",
        "▁k",
        "owa",
        "▁w",
        "aj",
        "en",
        "▁da",
        "fa",
        "▁mi",
        "yan",
        "▁gy",
        "ada",
        "▁da",
        "▁tu",
        "wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.5
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 20,
      "token_splits": [
        "▁K",
        "ak",
        "k",
        "ata",
        "▁t",
        "afi",
        "▁k",
        "owa",
        "▁w",
        "aj",
        "en",
        "▁da",
        "fa",
        "▁mi",
        "yan",
        "▁gy",
        "ada",
        "▁da",
        "▁tu",
        "wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.429
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 18,
      "token_splits": [
        "Ka",
        "##kka",
        "##ta",
        "ta",
        "##fi",
        "ko",
        "##wa",
        "wa",
        "##jen",
        "da",
        "##fa",
        "mi",
        "##yan",
        "g",
        "##yada",
        "da",
        "tu",
        "##wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.2
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Kakkata tafi kowa wajen dafa miyan gyada da tuwo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Kak",
        "kata",
        "▁ta",
        "fi",
        "▁",
        "kowa",
        "▁wajen",
        "▁da",
        "fa",
        "▁mi",
        "yan",
        "▁g",
        "yada",
        "▁da",
        "▁tu",
        "wo"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.067
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 25,
      "token_splits": [
        "N",
        "ne",
        " n",
        "ne",
        " m",
        " na",
        "-es",
        "i",
        " of",
        "e",
        " eg",
        "usi",
        " k",
        "ach",
        "asi",
        " m",
        "ma",
        " na",
        " ji",
        " ak",
        "p",
        "ụ",
        " ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.083
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 21,
      "token_splits": [
        "N",
        "ne",
        " nne",
        " m",
        " na",
        "-",
        "esi",
        " of",
        "e",
        " eg",
        "usi",
        " k",
        "ach",
        "asi",
        " mma",
        " na",
        " ji",
        " ak",
        "pụ",
        " ak",
        "pụ"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.75
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 25,
      "token_splits": [
        "N",
        "ne",
        "Ġn",
        "ne",
        "Ġm",
        "Ġna",
        "-es",
        "i",
        "Ġof",
        "e",
        "Ġeg",
        "usi",
        "Ġk",
        "ach",
        "asi",
        "Ġm",
        "ma",
        "Ġna",
        "Ġji",
        "Ġak",
        "p",
        "á»¥",
        "Ġak",
        "p",
        "á»¥"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.083
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 25,
      "token_splits": [
        "N",
        "ne",
        "▁n",
        "ne",
        "▁m",
        "▁na",
        "-",
        "esi",
        "▁of",
        "e",
        "▁e",
        "gu",
        "si",
        "▁k",
        "ach",
        "asi",
        "▁mma",
        "▁na",
        "▁ji",
        "▁ak",
        "p",
        "ụ",
        "▁ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 2.083
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 26,
      "token_splits": [
        "▁N",
        "ne",
        "▁n",
        "ne",
        "▁m",
        "▁na",
        "-",
        "esi",
        "▁of",
        "e",
        "▁eg",
        "us",
        "i",
        "▁k",
        "ach",
        "asi",
        "▁m",
        "ma",
        "▁na",
        "▁ji",
        "▁ak",
        "p",
        "ụ",
        "▁ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.857
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 26,
      "token_splits": [
        "N",
        "##ne",
        "nn",
        "##e",
        "m",
        "na",
        "-",
        "esi",
        "of",
        "##e",
        "e",
        "##gus",
        "##i",
        "ka",
        "##chas",
        "##i",
        "mm",
        "##a",
        "na",
        "ji",
        "ak",
        "##p",
        "##ụ",
        "ak",
        "##p",
        "##ụ"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.733
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Nne nne m na-esi ofe egusi kachasi mma na ji akpụ akpụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 24,
      "token_splits": [
        "▁N",
        "ne",
        "▁",
        "nne",
        "▁m",
        "▁na",
        "-",
        "esi",
        "▁of",
        "e",
        "▁e",
        "gusi",
        "▁ka",
        "chasi",
        "▁",
        "mma",
        "▁na",
        "▁ji",
        "▁ak",
        "p",
        "ụ",
        "▁ak",
        "p",
        "ụ"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 16,
      "token_splits": [
        "My",
        " grand",
        "m",
        "ama",
        " de",
        "y",
        " cook",
        " de",
        " best",
        " eg",
        "usi",
        " soup",
        " with",
        " pounded",
        " y",
        "am"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "My",
        " grand",
        "m",
        "ama",
        " dey",
        " cook",
        " de",
        " best",
        " eg",
        "usi",
        " soup",
        " with",
        " p",
        "ounded",
        " yam"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.25
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 16,
      "token_splits": [
        "My",
        "Ġgrand",
        "m",
        "ama",
        "Ġde",
        "y",
        "Ġcook",
        "Ġde",
        "Ġbest",
        "Ġeg",
        "usi",
        "Ġsoup",
        "Ġwith",
        "Ġpounded",
        "Ġy",
        "am"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 14,
      "token_splits": [
        "My",
        "▁grand",
        "mama",
        "▁dey",
        "▁cook",
        "▁de",
        "▁best",
        "▁e",
        "gu",
        "si",
        "▁soup",
        "▁with",
        "▁pounded",
        "▁yam"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 12,
      "token_ratio_vs_english": 1.167
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 18,
      "token_splits": [
        "▁My",
        "▁grand",
        "m",
        "ama",
        "▁de",
        "y",
        "▁cook",
        "▁de",
        "▁best",
        "▁eg",
        "us",
        "i",
        "▁soup",
        "▁with",
        "▁pound",
        "ed",
        "▁y",
        "am"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 20,
      "token_splits": [
        "My",
        "grand",
        "##ma",
        "##ma",
        "de",
        "##y",
        "co",
        "##ok",
        "de",
        "best",
        "e",
        "##gus",
        "##i",
        "sou",
        "##p",
        "with",
        "pound",
        "##ed",
        "ya",
        "##m"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.333
    },
    {
      "entry_id": 97,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "My grandmama dey cook de best egusi soup with pounded yam",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁My",
        "▁grand",
        "mama",
        "▁de",
        "y",
        "▁cook",
        "▁de",
        "▁best",
        "▁e",
        "gusi",
        "▁soup",
        "▁with",
        "▁p",
        "ounded",
        "▁ya",
        "m"
      ],
      "english_equivalent": "My grandmother makes the best egusi soup with pounded yam",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.067
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Young",
        " people",
        " in",
        " Nigeria",
        " are",
        " building",
        " innovative",
        " technology",
        " startups"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Young",
        " people",
        " in",
        " Nigeria",
        " are",
        " building",
        " innovative",
        " technology",
        " startups"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "Young",
        "Ġpeople",
        "Ġin",
        "ĠNigeria",
        "Ġare",
        "Ġbuilding",
        "Ġinnovative",
        "Ġtechnology",
        "Ġstartups"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "Young",
        "▁people",
        "▁in",
        "▁Nigeria",
        "▁are",
        "▁building",
        "▁innovative",
        "▁technology",
        "▁startups"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁Young",
        "▁people",
        "▁in",
        "▁Nigeria",
        "▁are",
        "▁building",
        "▁innovative",
        "▁technology",
        "▁start",
        "ups"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "Young",
        "people",
        "in",
        "Nigeria",
        "are",
        "building",
        "innovative",
        "technology",
        "startu",
        "##ps"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "Young people in Nigeria are building innovative technology startups",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁Young",
        "▁people",
        "▁in",
        "▁Nigeria",
        "▁are",
        "▁building",
        "▁innovative",
        "▁technology",
        "▁startup",
        "s"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 50,
      "token_splits": [
        "À",
        "w",
        "ọn",
        " ",
        "ọ",
        "̀",
        "d",
        "ọ",
        "́",
        " n",
        "í",
        " N",
        "à",
        "ì",
        "j",
        "í",
        "r",
        "í",
        "à",
        " ",
        "ń",
        " k",
        "ọ",
        "́",
        " à",
        "w",
        "ọn",
        " il",
        "é",
        " i",
        "�",
        "�",
        "�",
        "�",
        "�",
        "́",
        " �",
        "�",
        "m",
        "ọ",
        "̀",
        " ",
        "�",
        "�",
        "̀",
        "r",
        "ọ",
        " t",
        "unt",
        "un"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 5.556
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 30,
      "token_splits": [
        "À",
        "wọn",
        " ọ",
        "̀",
        "d",
        "ọ",
        "́",
        " ní",
        " N",
        "à",
        "ì",
        "j",
        "ír",
        "í",
        "à",
        " ń",
        " kọ",
        "́",
        " àwọn",
        " il",
        "é",
        " iṣẹ",
        "́",
        " ì",
        "mọ",
        "̀",
        " ẹ",
        "̀",
        "rọ",
        " tuntun"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 45,
      "token_splits": [
        "ÃĢ",
        "w",
        "á»įn",
        "Ġ",
        "á»į",
        "ÌĢ",
        "d",
        "á»į",
        "Ìģ",
        "ĠnÃŃ",
        "ĠN",
        "Ãł",
        "Ã¬",
        "jÃŃ",
        "r",
        "ÃŃ",
        "Ãł",
        "Ġ",
        "ÅĦ",
        "Ġk",
        "á»į",
        "Ìģ",
        "ĠÃł",
        "w",
        "á»įn",
        "Ġil",
        "Ã©",
        "Ġi",
        "á¹",
        "£",
        "áº¹",
        "Ìģ",
        "ĠÃ",
        "¬",
        "m",
        "á»į",
        "ÌĢ",
        "Ġ",
        "áº¹",
        "ÌĢ",
        "r",
        "á»į",
        "Ġt",
        "unt",
        "un"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 5.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 40,
      "token_splits": [
        "À",
        "w",
        "ọn",
        "▁ọ",
        "̀",
        "d",
        "ọ",
        "́",
        "▁ní",
        "▁N",
        "à",
        "ì",
        "jí",
        "rí",
        "à",
        "▁",
        "ń",
        "▁k",
        "ọ",
        "́",
        "▁à",
        "w",
        "ọn",
        "▁il",
        "é",
        "▁i",
        "ṣ",
        "ẹ",
        "́",
        "▁",
        "ìm",
        "ọ",
        "̀",
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁tunt",
        "un"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 4.444
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 51,
      "token_splits": [
        "▁À",
        "w",
        "ọ",
        "n",
        "▁",
        "ọ",
        "̀",
        "d",
        "ọ",
        "́",
        "▁n",
        "í",
        "▁N",
        "à",
        "ì",
        "j",
        "ír",
        "í",
        "à",
        "▁",
        "ń",
        "▁k",
        "ọ",
        "́",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁il",
        "é",
        "▁i",
        "ṣ",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁",
        "ì",
        "m",
        "ọ",
        "̀",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ọ",
        "▁t",
        "unt",
        "un"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 5.1
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 32,
      "token_splits": [
        "À",
        "##w",
        "##ọn",
        "ọ",
        "##̀",
        "##d",
        "##ọ́",
        "ní",
        "N",
        "##à",
        "##ì",
        "##jí",
        "##rí",
        "##à",
        "ń",
        "k",
        "##ọ́",
        "àwọn",
        "il",
        "##é",
        "i",
        "##ṣ",
        "##ẹ́",
        "ì",
        "##m",
        "##ọ̀",
        "ẹ",
        "##̀",
        "##r",
        "##ọ",
        "tun",
        "##tun"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.2
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àwọn ọ̀dọ́ ní Nàìjíríà ń kọ́ àwọn ilé iṣẹ́ ìmọ̀ ẹ̀rọ tuntun",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 44,
      "token_splits": [
        "▁À",
        "w",
        "ọ",
        "n",
        "▁",
        "ọ",
        "̀",
        "d",
        "ọ",
        "́",
        "▁ní",
        "▁N",
        "à",
        "ì",
        "jí",
        "rí",
        "à",
        "▁",
        "ń",
        "▁k",
        "ọ",
        "́",
        "▁à",
        "w",
        "ọ",
        "n",
        "▁i",
        "lé",
        "▁i",
        "ṣ",
        "ẹ",
        "́",
        "▁",
        "ì",
        "m",
        "ọ",
        "̀",
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁tuntu",
        "n"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 4.4
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 19,
      "token_splits": [
        "Mat",
        "asa",
        " a",
        " N",
        "ij",
        "eri",
        "ya",
        " sun",
        "a",
        " g",
        "ina",
        " sab",
        "bin",
        " kam",
        "fan",
        "on",
        "in",
        " fas",
        "aha"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.111
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Mat",
        "asa",
        " a",
        " Nij",
        "eriya",
        " suna",
        " g",
        "ina",
        " sab",
        "bin",
        " kam",
        "fan",
        "onin",
        " fas",
        "aha"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 19,
      "token_splits": [
        "Mat",
        "asa",
        "Ġa",
        "ĠN",
        "ij",
        "eri",
        "ya",
        "Ġsun",
        "a",
        "Ġg",
        "ina",
        "Ġsab",
        "bin",
        "Ġkam",
        "fan",
        "on",
        "in",
        "Ġfas",
        "aha"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.111
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 17,
      "token_splits": [
        "Mat",
        "asa",
        "▁a",
        "▁Nij",
        "eri",
        "ya",
        "▁suna",
        "▁g",
        "ina",
        "▁sab",
        "bin",
        "▁kam",
        "fan",
        "onin",
        "▁f",
        "asa",
        "ha"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.889
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 20,
      "token_splits": [
        "▁Mat",
        "asa",
        "▁a",
        "▁N",
        "ij",
        "eri",
        "ya",
        "▁sun",
        "a",
        "▁g",
        "ina",
        "▁sab",
        "bin",
        "▁kam",
        "fan",
        "on",
        "in",
        "▁f",
        "as",
        "aha"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 19,
      "token_splits": [
        "Mata",
        "##sa",
        "a",
        "Nije",
        "##riy",
        "##a",
        "sun",
        "##a",
        "gi",
        "##na",
        "sa",
        "##bbi",
        "##n",
        "kam",
        "##fano",
        "##nin",
        "fa",
        "##sah",
        "##a"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.9
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Matasa a Nijeriya suna gina sabbin kamfanonin fasaha",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁Mata",
        "sa",
        "▁a",
        "▁Nije",
        "riya",
        "▁suna",
        "▁gina",
        "▁sab",
        "bin",
        "▁kam",
        "fano",
        "nin",
        "▁fasa",
        "ha"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 38,
      "token_splits": [
        "�",
        "�",
        "m",
        "ụ",
        " ok",
        "or",
        "ob",
        "ị",
        "a",
        " na",
        " Na",
        "ị",
        "j",
        "ir",
        "ị",
        "a",
        " na",
        "-",
        "ew",
        "u",
        " ",
        "ụ",
        "l",
        "ọ",
        " ",
        "ọ",
        "r",
        "ụ",
        " tekn",
        "ụ",
        "z",
        "ụ",
        " ",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 4.222
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 20,
      "token_splits": [
        "Ụ",
        "mụ",
        " ok",
        "or",
        "ob",
        "ịa",
        " na",
        " Na",
        "ị",
        "jir",
        "ịa",
        " na",
        "-ew",
        "u",
        " ụlọ",
        " ọrụ",
        " tekn",
        "ụ",
        "zụ",
        " ọhụrụ"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.222
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 36,
      "token_splits": [
        "á»",
        "¤",
        "m",
        "á»¥",
        "Ġok",
        "or",
        "ob",
        "á»ĭa",
        "Ġna",
        "ĠNa",
        "á»ĭ",
        "j",
        "ir",
        "á»ĭa",
        "Ġna",
        "-",
        "ew",
        "u",
        "Ġ",
        "á»¥",
        "l",
        "á»į",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        "Ġtekn",
        "á»¥",
        "z",
        "á»¥",
        "Ġ",
        "á»į",
        "h",
        "á»¥",
        "r",
        "á»¥"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 4.0
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 33,
      "token_splits": [
        "Ụ",
        "m",
        "ụ",
        "▁ok",
        "or",
        "ob",
        "ịa",
        "▁na",
        "▁Na",
        "ị",
        "jir",
        "ịa",
        "▁na",
        "-",
        "ew",
        "u",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁ọ",
        "r",
        "ụ",
        "▁tek",
        "n",
        "ụ",
        "z",
        "ụ",
        "▁ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.667
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 41,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0xA4>",
        "m",
        "ụ",
        "▁ok",
        "or",
        "ob",
        "ị",
        "a",
        "▁na",
        "▁Na",
        "ị",
        "j",
        "ir",
        "ị",
        "a",
        "▁na",
        "-",
        "ew",
        "u",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁te",
        "kn",
        "ụ",
        "z",
        "ụ",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 4.1
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 32,
      "token_splits": [
        "[UNK]",
        "oko",
        "##ro",
        "##b",
        "##ị",
        "##a",
        "na",
        "Na",
        "##ị",
        "##jir",
        "##ị",
        "##a",
        "na",
        "-",
        "e",
        "##wu",
        "ụ",
        "##l",
        "##ọ",
        "ọ",
        "##r",
        "##ụ",
        "tek",
        "##n",
        "##ụ",
        "##z",
        "##ụ",
        "ọ",
        "##h",
        "##ụ",
        "##r",
        "##ụ"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.2
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ụmụ okorobịa na Naịjirịa na-ewu ụlọ ọrụ teknụzụ ọhụrụ",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 37,
      "token_splits": [
        "▁",
        "Ụ",
        "m",
        "ụ",
        "▁oko",
        "rob",
        "ị",
        "a",
        "▁na",
        "▁Na",
        "ị",
        "jir",
        "ị",
        "a",
        "▁na",
        "-",
        "e",
        "wu",
        "▁",
        "ụ",
        "l",
        "ọ",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁tek",
        "n",
        "ụ",
        "z",
        "ụ",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.7
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Young",
        " people",
        " for",
        " Nigeria",
        " de",
        "y",
        " build",
        " new",
        " new",
        " tech",
        " companies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "Young",
        " people",
        " for",
        " Nigeria",
        " dey",
        " build",
        " new",
        " new",
        " tech",
        " companies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "Young",
        "Ġpeople",
        "Ġfor",
        "ĠNigeria",
        "Ġde",
        "y",
        "Ġbuild",
        "Ġnew",
        "Ġnew",
        "Ġtech",
        "Ġcompanies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "Young",
        "▁people",
        "▁for",
        "▁Nigeria",
        "▁dey",
        "▁build",
        "▁new",
        "▁new",
        "▁tech",
        "▁companies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.111
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Young",
        "▁people",
        "▁for",
        "▁Nigeria",
        "▁de",
        "y",
        "▁build",
        "▁new",
        "▁new",
        "▁tech",
        "▁companies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Young",
        "people",
        "for",
        "Nigeria",
        "de",
        "##y",
        "build",
        "new",
        "new",
        "tech",
        "companies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 98,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Young people for Nigeria dey build new new tech companies",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Young",
        "▁people",
        "▁for",
        "▁Nigeria",
        "▁de",
        "y",
        "▁build",
        "▁new",
        "▁new",
        "▁tech",
        "▁companies"
      ],
      "english_equivalent": "Young people in Nigeria are building innovative technology startups",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Happy",
        " New",
        " Year",
        ",",
        " may",
        " this",
        " year",
        " bring",
        " prosperity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 9,
      "token_splits": [
        "Happy",
        " New",
        " Year",
        ",",
        " may",
        " this",
        " year",
        " bring",
        " prosperity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 9,
      "token_splits": [
        "Happy",
        "ĠNew",
        "ĠYear",
        ",",
        "Ġmay",
        "Ġthis",
        "Ġyear",
        "Ġbring",
        "Ġprosperity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 9,
      "token_splits": [
        "Happy",
        "▁New",
        "▁Year",
        ",",
        "▁may",
        "▁this",
        "▁year",
        "▁bring",
        "▁prosperity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁Happy",
        "▁New",
        "▁Year",
        ",",
        "▁may",
        "▁this",
        "▁year",
        "▁bring",
        "▁prosper",
        "ity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Happy",
        "New",
        "Year",
        ",",
        "may",
        "this",
        "year",
        "bring",
        "pro",
        "##sper",
        "##ity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "eng",
      "language_name": "English",
      "text": "Happy New Year, may this year bring prosperity",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁Happy",
        "▁New",
        "▁Year",
        ",",
        "▁may",
        "▁this",
        "▁year",
        "▁bring",
        "▁prosper",
        "ity"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 30,
      "token_splits": [
        "�",
        "�",
        " k",
        "ú",
        " ",
        "ọ",
        "d",
        "ún",
        " t",
        "unt",
        "un",
        ",",
        " k",
        "í",
        " ",
        "ọ",
        "d",
        "ún",
        " y",
        "ì",
        "í",
        " mú",
        " à",
        "l",
        "à",
        "áf",
        "í",
        "à",
        " w",
        "á"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 22,
      "token_splits": [
        "Ẹ",
        " kú",
        " ọd",
        "ún",
        " tuntun",
        ",",
        " kí",
        " ọd",
        "ún",
        " y",
        "ì",
        "í",
        " mú",
        " à",
        "l",
        "à",
        "á",
        "f",
        "í",
        "à",
        " w",
        "á"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.444
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 30,
      "token_splits": [
        "áº",
        "¸",
        "Ġk",
        "Ãº",
        "Ġ",
        "á»į",
        "d",
        "Ãºn",
        "Ġt",
        "unt",
        "un",
        ",",
        "Ġk",
        "ÃŃ",
        "Ġ",
        "á»į",
        "d",
        "Ãºn",
        "Ġy",
        "Ã¬",
        "ÃŃ",
        "ĠmÃº",
        "ĠÃł",
        "l",
        "Ãł",
        "Ã¡f",
        "ÃŃ",
        "Ãł",
        "Ġw",
        "Ã¡"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.333
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 23,
      "token_splits": [
        "Ẹ",
        "▁kú",
        "▁ọ",
        "d",
        "ún",
        "▁tunt",
        "un",
        ",",
        "▁kí",
        "▁ọ",
        "d",
        "ún",
        "▁y",
        "ì",
        "í",
        "▁mú",
        "▁à",
        "là",
        "á",
        "fí",
        "à",
        "▁w",
        "á"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.556
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 34,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB8>",
        "▁k",
        "ú",
        "▁",
        "ọ",
        "d",
        "ún",
        "▁t",
        "unt",
        "un",
        ",",
        "▁k",
        "í",
        "▁",
        "ọ",
        "d",
        "ún",
        "▁y",
        "ì",
        "í",
        "▁m",
        "ú",
        "▁à",
        "l",
        "à",
        "á",
        "f",
        "í",
        "à",
        "▁w",
        "á"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.4
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 26,
      "token_splits": [
        "Ẹ",
        "k",
        "##ú",
        "ọ",
        "##dú",
        "##n",
        "tun",
        "##tun",
        ",",
        "k",
        "##í",
        "ọ",
        "##dú",
        "##n",
        "y",
        "##ìí",
        "m",
        "##ú",
        "à",
        "##là",
        "##á",
        "##f",
        "##í",
        "##à",
        "w",
        "##á"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.364
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Ẹ kú ọdún tuntun, kí ọdún yìí mú àlàáfíà wá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 26,
      "token_splits": [
        "▁",
        "Ẹ",
        "▁kú",
        "▁",
        "ọ",
        "dú",
        "n",
        "▁tuntu",
        "n",
        ",",
        "▁kí",
        "▁",
        "ọ",
        "dú",
        "n",
        "▁y",
        "ì",
        "í",
        "▁mú",
        "▁à",
        "là",
        "á",
        "fí",
        "à",
        "▁w",
        "á"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.6
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 21,
      "token_splits": [
        "B",
        "ark",
        "a",
        " da",
        " sab",
        "u",
        "war",
        " she",
        "k",
        "ara",
        ",",
        " Allah",
        " ya",
        " sa",
        " she",
        "k",
        "ara",
        " mai",
        " alb",
        "ark",
        "a"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.333
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 14,
      "token_splits": [
        "B",
        "arka",
        " da",
        " sab",
        "uwar",
        " shekara",
        ",",
        " Allah",
        " ya",
        " sa",
        " shekara",
        " mai",
        " alb",
        "arka"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.556
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 19,
      "token_splits": [
        "B",
        "ark",
        "a",
        "Ġda",
        "Ġsab",
        "u",
        "war",
        "Ġshe",
        "kara",
        ",",
        "ĠAllah",
        "Ġya",
        "Ġsa",
        "Ġshe",
        "kara",
        "Ġmai",
        "Ġalb",
        "ark",
        "a"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.111
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 17,
      "token_splits": [
        "B",
        "arka",
        "▁da",
        "▁sab",
        "u",
        "war",
        "▁she",
        "kara",
        ",",
        "▁Allah",
        "▁ya",
        "▁sa",
        "▁she",
        "kara",
        "▁mai",
        "▁alb",
        "arka"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.889
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 22,
      "token_splits": [
        "▁Bark",
        "a",
        "▁da",
        "▁sab",
        "u",
        "war",
        "▁she",
        "k",
        "ara",
        ",",
        "▁All",
        "ah",
        "▁ya",
        "▁sa",
        "▁she",
        "k",
        "ara",
        "▁mai",
        "▁al",
        "b",
        "ark",
        "a"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.2
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 17,
      "token_splits": [
        "Bar",
        "##ka",
        "da",
        "sa",
        "##bu",
        "##war",
        "shek",
        "##ara",
        ",",
        "Allah",
        "ya",
        "sa",
        "shek",
        "##ara",
        "mai",
        "alba",
        "##rka"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.545
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Barka da sabuwar shekara, Allah ya sa shekara mai albarka",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 14,
      "token_splits": [
        "▁Bar",
        "ka",
        "▁da",
        "▁s",
        "abuwar",
        "▁shekara",
        ",",
        "▁Allah",
        "▁ya",
        "▁sa",
        "▁shekara",
        "▁mai",
        "▁alba",
        "rka"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.4
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 27,
      "token_splits": [
        "�",
        "�",
        "n",
        "ụ",
        " ",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        " ",
        "ọ",
        "ma",
        ",",
        " ka",
        " af",
        "ọ",
        " a",
        " wet",
        "ara",
        " any",
        "ị",
        " ",
        "ọ",
        "g",
        "ani",
        "hu"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Ọ",
        "n",
        "ụ",
        " ọhụrụ",
        " ọma",
        ",",
        " ka",
        " afọ",
        " a",
        " wet",
        "ara",
        " anyị",
        " ọ",
        "gan",
        "ihu"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.667
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 27,
      "token_splits": [
        "á»",
        "Į",
        "n",
        "á»¥",
        "Ġ",
        "á»į",
        "h",
        "á»¥",
        "r",
        "á»¥",
        "Ġ",
        "á»į",
        "ma",
        ",",
        "Ġka",
        "Ġaf",
        "á»į",
        "Ġa",
        "Ġwet",
        "ara",
        "Ġany",
        "á»ĭ",
        "Ġ",
        "á»į",
        "g",
        "ani",
        "hu"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 22,
      "token_splits": [
        "Ọ",
        "n",
        "ụ",
        "▁ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁ọ",
        "ma",
        ",",
        "▁ka",
        "▁af",
        "ọ",
        "▁a",
        "▁wet",
        "ara",
        "▁any",
        "ị",
        "▁ọ",
        "gani",
        "hu"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 2.444
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 29,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBB>",
        "<0x8C>",
        "n",
        "ụ",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁",
        "ọ",
        "ma",
        ",",
        "▁ka",
        "▁af",
        "ọ",
        "▁a",
        "▁wet",
        "ara",
        "▁any",
        "ị",
        "▁",
        "ọ",
        "gan",
        "ih",
        "u"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.9
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 22,
      "token_splits": [
        "Ọ",
        "##n",
        "##ụ",
        "ọ",
        "##h",
        "##ụ",
        "##r",
        "##ụ",
        "ọ",
        "##ma",
        ",",
        "ka",
        "af",
        "##ọ",
        "a",
        "wet",
        "##ara",
        "any",
        "##ị",
        "ọ",
        "##gani",
        "##hu"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Ọnụ ọhụrụ ọma, ka afọ a wetara anyị ọganihu",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 26,
      "token_splits": [
        "▁",
        "Ọ",
        "n",
        "ụ",
        "▁",
        "ọ",
        "h",
        "ụ",
        "r",
        "ụ",
        "▁",
        "ọ",
        "ma",
        ",",
        "▁ka",
        "▁af",
        "ọ",
        "▁a",
        "▁w",
        "etara",
        "▁any",
        "ị",
        "▁",
        "ọ",
        "gani",
        "hu"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 2.6
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Happy",
        " New",
        " Year",
        " o",
        ",",
        " make",
        " dis",
        " year",
        " bring",
        " better",
        " thing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "Happy",
        " New",
        " Year",
        " o",
        ",",
        " make",
        " dis",
        " year",
        " bring",
        " better",
        " thing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "Happy",
        "ĠNew",
        "ĠYear",
        "Ġo",
        ",",
        "Ġmake",
        "Ġdis",
        "Ġyear",
        "Ġbring",
        "Ġbetter",
        "Ġthing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 11,
      "token_splits": [
        "Happy",
        "▁New",
        "▁Year",
        "▁o",
        ",",
        "▁make",
        "▁dis",
        "▁year",
        "▁bring",
        "▁better",
        "▁thing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 9,
      "token_ratio_vs_english": 1.222
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 11,
      "token_splits": [
        "▁Happy",
        "▁New",
        "▁Year",
        "▁o",
        ",",
        "▁make",
        "▁dis",
        "▁year",
        "▁bring",
        "▁better",
        "▁thing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 11,
      "token_splits": [
        "Happy",
        "New",
        "Year",
        "o",
        ",",
        "make",
        "dis",
        "year",
        "bring",
        "better",
        "thing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 99,
      "category": "greetings",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "Happy New Year o, make dis year bring better thing",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 11,
      "token_splits": [
        "▁Happy",
        "▁New",
        "▁Year",
        "▁o",
        ",",
        "▁make",
        "▁dis",
        "▁year",
        "▁bring",
        "▁better",
        "▁thing"
      ],
      "english_equivalent": "Happy New Year, may this year bring prosperity",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.1
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 11,
      "token_splits": [
        "The",
        " farmer",
        " planted",
        " cass",
        "ava",
        " and",
        " maize",
        " before",
        " the",
        " rains",
        " came"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 11,
      "token_splits": [
        "The",
        " farmer",
        " planted",
        " cass",
        "ava",
        " and",
        " maize",
        " before",
        " the",
        " rains",
        " came"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 11,
      "token_splits": [
        "The",
        "Ġfarmer",
        "Ġplanted",
        "Ġcass",
        "ava",
        "Ġand",
        "Ġmaize",
        "Ġbefore",
        "Ġthe",
        "Ġrains",
        "Ġcame"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "The",
        "▁farmer",
        "▁planted",
        "▁cassava",
        "▁and",
        "▁maize",
        "▁before",
        "▁the",
        "▁rains",
        "▁came"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 14,
      "token_splits": [
        "▁The",
        "▁farmer",
        "▁planted",
        "▁c",
        "ass",
        "ava",
        "▁and",
        "▁ma",
        "ize",
        "▁before",
        "▁the",
        "▁r",
        "ains",
        "▁came"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 14,
      "token_splits": [
        "The",
        "farmer",
        "planted",
        "cas",
        "##sa",
        "##va",
        "and",
        "mai",
        "##ze",
        "before",
        "the",
        "rain",
        "##s",
        "came"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "eng",
      "language_name": "English",
      "text": "The farmer planted cassava and maize before the rains came",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 15,
      "token_splits": [
        "▁The",
        "▁farmer",
        "▁plante",
        "d",
        "▁ca",
        "ssa",
        "va",
        "▁and",
        "▁mai",
        "ze",
        "▁before",
        "▁the",
        "▁rain",
        "s",
        "▁came"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 41,
      "token_splits": [
        "À",
        "gb",
        "�",
        "�",
        "̀",
        " n",
        "á",
        "à",
        " gb",
        "ì",
        "n",
        " ",
        "�",
        "�",
        "̀",
        "g",
        "�",
        "�",
        "́",
        " à",
        "ti",
        " à",
        "gb",
        "à",
        "do",
        " k",
        "í",
        " �",
        "�",
        "j",
        "ò",
        " t",
        "ó",
        " b",
        "�",
        "�",
        "̀",
        "r",
        "�",
        "�",
        "̀"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.727
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 24,
      "token_splits": [
        "À",
        "gbẹ",
        "̀",
        " náà",
        " gb",
        "ìn",
        " ẹ",
        "̀",
        "gẹ",
        "́",
        " àti",
        " à",
        "gb",
        "à",
        "do",
        " kí",
        " ò",
        "j",
        "ò",
        " tó",
        " bẹ",
        "̀",
        "rẹ",
        "̀"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 2.182
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 34,
      "token_splits": [
        "ÃĢ",
        "gb",
        "áº¹",
        "ÌĢ",
        "ĠnÃ¡",
        "Ãł",
        "Ġgb",
        "Ã¬n",
        "Ġ",
        "áº¹",
        "ÌĢ",
        "g",
        "áº¹",
        "Ìģ",
        "ĠÃł",
        "ti",
        "ĠÃł",
        "gb",
        "Ãł",
        "do",
        "Ġk",
        "ÃŃ",
        "ĠÃ",
        "²",
        "j",
        "Ã²",
        "Ġt",
        "Ã³",
        "Ġb",
        "áº¹",
        "ÌĢ",
        "r",
        "áº¹",
        "ÌĢ"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.091
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 31,
      "token_splits": [
        "À",
        "gb",
        "ẹ",
        "̀",
        "▁ná",
        "à",
        "▁gb",
        "ìn",
        "▁",
        "ẹ",
        "̀",
        "g",
        "ẹ",
        "́",
        "▁à",
        "ti",
        "▁à",
        "gb",
        "à",
        "do",
        "▁kí",
        "▁ò",
        "j",
        "ò",
        "▁tó",
        "▁b",
        "ẹ",
        "̀",
        "r",
        "ẹ",
        "̀"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.1
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 46,
      "token_splits": [
        "▁À",
        "gb",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "▁ná",
        "à",
        "▁g",
        "b",
        "ì",
        "n",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "g",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "́",
        "▁à",
        "ti",
        "▁à",
        "gb",
        "à",
        "do",
        "▁k",
        "í",
        "▁",
        "ò",
        "j",
        "ò",
        "▁t",
        "ó",
        "▁b",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 3.286
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 28,
      "token_splits": [
        "À",
        "##g",
        "##b",
        "##ẹ̀",
        "náà",
        "g",
        "##b",
        "##ì",
        "##n",
        "ẹ",
        "##̀",
        "##g",
        "##ẹ́",
        "àti",
        "à",
        "##g",
        "##bà",
        "##do",
        "k",
        "##í",
        "ò",
        "##j",
        "##ò",
        "tó",
        "b",
        "##ẹ̀",
        "##r",
        "##ẹ̀"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "yor",
      "language_name": "Yorùbá",
      "text": "Àgbẹ̀ náà gbìn ẹ̀gẹ́ àti àgbàdo kí òjò tó bẹ̀rẹ̀",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 34,
      "token_splits": [
        "▁À",
        "gb",
        "ẹ",
        "̀",
        "▁ná",
        "à",
        "▁",
        "gb",
        "ì",
        "n",
        "▁",
        "ẹ",
        "̀",
        "g",
        "ẹ",
        "́",
        "▁à",
        "ti",
        "▁à",
        "gb",
        "à",
        "do",
        "▁kí",
        "▁",
        "ò",
        "j",
        "ò",
        "▁tó",
        "▁b",
        "ẹ",
        "̀",
        "r",
        "ẹ",
        "̀"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.267
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 17,
      "token_splits": [
        "Man",
        "omi",
        " ya",
        " sh",
        "uka",
        " ro",
        "go",
        " da",
        " mas",
        "ara",
        " k",
        "af",
        "in",
        " dam",
        "ina",
        " ta",
        " zo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.545
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 15,
      "token_splits": [
        "Man",
        "omi",
        " ya",
        " sh",
        "uka",
        " ro",
        "go",
        " da",
        " mas",
        "ara",
        " kafin",
        " dam",
        "ina",
        " ta",
        " zo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.364
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 16,
      "token_splits": [
        "Man",
        "omi",
        "Ġya",
        "Ġsh",
        "uka",
        "Ġro",
        "go",
        "Ġda",
        "Ġmas",
        "ara",
        "Ġkaf",
        "in",
        "Ġdam",
        "ina",
        "Ġta",
        "Ġzo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.455
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 16,
      "token_splits": [
        "Man",
        "omi",
        "▁ya",
        "▁sh",
        "uka",
        "▁ro",
        "go",
        "▁da",
        "▁mas",
        "ara",
        "▁kaf",
        "in",
        "▁dam",
        "ina",
        "▁ta",
        "▁zo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.6
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 18,
      "token_splits": [
        "▁Man",
        "omi",
        "▁ya",
        "▁sh",
        "u",
        "ka",
        "▁ro",
        "go",
        "▁da",
        "▁mas",
        "ara",
        "▁k",
        "af",
        "in",
        "▁dam",
        "ina",
        "▁ta",
        "▁zo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.286
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 16,
      "token_splits": [
        "Man",
        "##omi",
        "ya",
        "shu",
        "##ka",
        "ro",
        "##go",
        "da",
        "masa",
        "##ra",
        "ka",
        "##fin",
        "dam",
        "##ina",
        "ta",
        "zo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 1.143
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "hau",
      "language_name": "Hausa",
      "text": "Manomi ya shuka rogo da masara kafin damina ta zo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 16,
      "token_splits": [
        "▁Mano",
        "mi",
        "▁ya",
        "▁shu",
        "ka",
        "▁rog",
        "o",
        "▁da",
        "▁masa",
        "ra",
        "▁ka",
        "fin",
        "▁da",
        "mina",
        "▁ta",
        "▁zo"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 15,
      "token_ratio_vs_english": 1.067
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 34,
      "token_splits": [
        "On",
        "ye",
        " ",
        "ọ",
        "r",
        "ụ",
        " ug",
        "bo",
        " k",
        "ụ",
        "r",
        "ụ",
        " ak",
        "p",
        "ụ",
        " na",
        " ",
        "ọ",
        "ka",
        " tup",
        "u",
        " mm",
        "ị",
        "r",
        "ị",
        " ",
        "ọ",
        "z",
        "ụ",
        "z",
        "ọ",
        " ab",
        "ị",
        "a"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.091
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 20,
      "token_splits": [
        "On",
        "ye",
        " ọrụ",
        " ug",
        "bo",
        " k",
        "ụrụ",
        " ak",
        "pụ",
        " na",
        " ọ",
        "ka",
        " tupu",
        " mm",
        "ịrị",
        " ọ",
        "zụ",
        "zọ",
        " ab",
        "ịa"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 1.818
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 33,
      "token_splits": [
        "On",
        "ye",
        "Ġ",
        "á»į",
        "r",
        "á»¥",
        "Ġug",
        "bo",
        "Ġk",
        "á»¥",
        "r",
        "á»¥",
        "Ġak",
        "p",
        "á»¥",
        "Ġna",
        "Ġ",
        "á»į",
        "ka",
        "Ġtup",
        "u",
        "Ġmm",
        "á»ĭ",
        "r",
        "á»ĭ",
        "Ġ",
        "á»į",
        "z",
        "á»¥",
        "z",
        "á»į",
        "Ġab",
        "á»ĭa"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 30,
      "token_splits": [
        "O",
        "nye",
        "▁ọ",
        "r",
        "ụ",
        "▁ug",
        "bo",
        "▁k",
        "ụ",
        "r",
        "ụ",
        "▁ak",
        "p",
        "ụ",
        "▁na",
        "▁ọ",
        "ka",
        "▁t",
        "upu",
        "▁mm",
        "ị",
        "r",
        "ị",
        "▁ọ",
        "z",
        "ụ",
        "z",
        "ọ",
        "▁ab",
        "ịa"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 10,
      "token_ratio_vs_english": 3.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 36,
      "token_splits": [
        "▁On",
        "ye",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁u",
        "g",
        "bo",
        "▁k",
        "ụ",
        "r",
        "ụ",
        "▁ak",
        "p",
        "ụ",
        "▁na",
        "▁",
        "ọ",
        "ka",
        "▁t",
        "up",
        "u",
        "▁mm",
        "ị",
        "r",
        "ị",
        "▁",
        "ọ",
        "z",
        "ụ",
        "z",
        "ọ",
        "▁ab",
        "ị",
        "a"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.571
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 31,
      "token_splits": [
        "On",
        "##ye",
        "ọ",
        "##r",
        "##ụ",
        "ug",
        "##bo",
        "k",
        "##ụ",
        "##r",
        "##ụ",
        "ak",
        "##p",
        "##ụ",
        "na",
        "ọ",
        "##ka",
        "tu",
        "##pu",
        "mm",
        "##ị",
        "##r",
        "##ị",
        "ọ",
        "##z",
        "##ụ",
        "##z",
        "##ọ",
        "ab",
        "##ị",
        "##a"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 2.214
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "ibo",
      "language_name": "Igbo",
      "text": "Onye ọrụ ugbo kụrụ akpụ na ọka tupu mmịrị ọzụzọ abịa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 34,
      "token_splits": [
        "▁O",
        "nye",
        "▁",
        "ọ",
        "r",
        "ụ",
        "▁ug",
        "bo",
        "▁k",
        "ụ",
        "r",
        "ụ",
        "▁ak",
        "p",
        "ụ",
        "▁na",
        "▁",
        "ọ",
        "ka",
        "▁tu",
        "pu",
        "▁mm",
        "ị",
        "r",
        "ị",
        "▁",
        "ọ",
        "z",
        "ụ",
        "z",
        "ọ",
        "▁ab",
        "ị",
        "a"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 15,
      "token_ratio_vs_english": 2.267
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "n_tokens": 10,
      "token_splits": [
        "De",
        " farmer",
        " plant",
        " gar",
        "ri",
        " and",
        " corn",
        " before",
        " rain",
        " start"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 0.909
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "n_tokens": 10,
      "token_splits": [
        "De",
        " farmer",
        " plant",
        " gar",
        "ri",
        " and",
        " corn",
        " before",
        " rain",
        " start"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 0.909
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "n_tokens": 10,
      "token_splits": [
        "De",
        "Ġfarmer",
        "Ġplant",
        "Ġgar",
        "ri",
        "Ġand",
        "Ġcorn",
        "Ġbefore",
        "Ġrain",
        "Ġstart"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 11,
      "token_ratio_vs_english": 0.909
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "n_tokens": 10,
      "token_splits": [
        "De",
        "▁farmer",
        "▁plant",
        "▁gar",
        "ri",
        "▁and",
        "▁corn",
        "▁before",
        "▁rain",
        "▁start"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 10,
      "token_ratio_vs_english": 1.0
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "n_tokens": 10,
      "token_splits": [
        "▁De",
        "▁farmer",
        "▁plant",
        "▁gar",
        "ri",
        "▁and",
        "▁corn",
        "▁before",
        "▁rain",
        "▁start"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 0.714
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "n_tokens": 10,
      "token_splits": [
        "De",
        "farmer",
        "plant",
        "gar",
        "##ri",
        "and",
        "corn",
        "before",
        "rain",
        "start"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 14,
      "token_ratio_vs_english": 0.714
    },
    {
      "entry_id": 100,
      "category": "sentences",
      "language": "pcm",
      "language_name": "Pidgin",
      "text": "De farmer plant garri and corn before rain start",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "n_tokens": 10,
      "token_splits": [
        "▁De",
        "▁farmer",
        "▁plant",
        "▁",
        "garri",
        "▁and",
        "▁corn",
        "▁before",
        "▁rain",
        "▁start"
      ],
      "english_equivalent": "The farmer planted cassava and maize before the rains came",
      "english_tokens": 15,
      "token_ratio_vs_english": 0.667
    }
  ],
  "aggregate_stats": [
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "mean_tokens_per_entry": 4.47,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "mean_tokens_per_entry": 4.05,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "mean_tokens_per_entry": 4.35,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "mean_tokens_per_entry": 4.25,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "mean_tokens_per_entry": 4.34,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "mean_tokens_per_entry": 4.47,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "eng",
      "language_name": "English",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "mean_tokens_per_entry": 4.55,
      "mean_token_ratio_vs_english": 1.0,
      "cost_multiplier_vs_english": 1.0,
      "estimated_naira_per_1m_equiv": 4000.0,
      "worst_word": "Good morning",
      "worst_word_tokens": 2,
      "worst_word_ratio": 1.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "mean_tokens_per_entry": 6.83,
      "mean_token_ratio_vs_english": 1.639,
      "cost_multiplier_vs_english": 1.528,
      "estimated_naira_per_1m_equiv": 6112.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 6,
      "worst_word_ratio": 6.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "mean_tokens_per_entry": 6.55,
      "mean_token_ratio_vs_english": 1.707,
      "cost_multiplier_vs_english": 1.617,
      "estimated_naira_per_1m_equiv": 6469.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 5,
      "worst_word_ratio": 5.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "mean_tokens_per_entry": 7.78,
      "mean_token_ratio_vs_english": 1.866,
      "cost_multiplier_vs_english": 1.789,
      "estimated_naira_per_1m_equiv": 7154.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 6,
      "worst_word_ratio": 6.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "mean_tokens_per_entry": 6.16,
      "mean_token_ratio_vs_english": 1.58,
      "cost_multiplier_vs_english": 1.449,
      "estimated_naira_per_1m_equiv": 5798.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 5,
      "worst_word_ratio": 5.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "mean_tokens_per_entry": 7.63,
      "mean_token_ratio_vs_english": 1.829,
      "cost_multiplier_vs_english": 1.758,
      "estimated_naira_per_1m_equiv": 7032.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 6,
      "worst_word_ratio": 6.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "mean_tokens_per_entry": 8.39,
      "mean_token_ratio_vs_english": 2.098,
      "cost_multiplier_vs_english": 1.877,
      "estimated_naira_per_1m_equiv": 7508.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 7,
      "worst_word_ratio": 7.0
    },
    {
      "language": "hau",
      "language_name": "Hausa",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "mean_tokens_per_entry": 6.11,
      "mean_token_ratio_vs_english": 1.531,
      "cost_multiplier_vs_english": 1.343,
      "estimated_naira_per_1m_equiv": 5371.0,
      "worst_word": "filin jirgin sama",
      "worst_word_tokens": 5,
      "worst_word_ratio": 5.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "mean_tokens_per_entry": 9.63,
      "mean_token_ratio_vs_english": 2.275,
      "cost_multiplier_vs_english": 2.154,
      "estimated_naira_per_1m_equiv": 8617.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 9,
      "worst_word_ratio": 9.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "mean_tokens_per_entry": 9.18,
      "mean_token_ratio_vs_english": 2.251,
      "cost_multiplier_vs_english": 2.267,
      "estimated_naira_per_1m_equiv": 9067.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 9,
      "worst_word_ratio": 9.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "mean_tokens_per_entry": 10.33,
      "mean_token_ratio_vs_english": 2.393,
      "cost_multiplier_vs_english": 2.375,
      "estimated_naira_per_1m_equiv": 9499.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 10,
      "worst_word_ratio": 10.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "mean_tokens_per_entry": 6.55,
      "mean_token_ratio_vs_english": 1.611,
      "cost_multiplier_vs_english": 1.541,
      "estimated_naira_per_1m_equiv": 6165.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 5,
      "worst_word_ratio": 5.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "mean_tokens_per_entry": 10.08,
      "mean_token_ratio_vs_english": 2.343,
      "cost_multiplier_vs_english": 2.323,
      "estimated_naira_per_1m_equiv": 9290.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 10,
      "worst_word_ratio": 10.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "mean_tokens_per_entry": 11.24,
      "mean_token_ratio_vs_english": 2.777,
      "cost_multiplier_vs_english": 2.515,
      "estimated_naira_per_1m_equiv": 10058.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 10,
      "worst_word_ratio": 10.0
    },
    {
      "language": "ibo",
      "language_name": "Igbo",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "mean_tokens_per_entry": 10.26,
      "mean_token_ratio_vs_english": 2.456,
      "cost_multiplier_vs_english": 2.255,
      "estimated_naira_per_1m_equiv": 9020.0,
      "worst_word": "mmịrị ọzụzọ",
      "worst_word_tokens": 10,
      "worst_word_ratio": 10.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "mean_tokens_per_entry": 4.83,
      "mean_token_ratio_vs_english": 1.062,
      "cost_multiplier_vs_english": 1.081,
      "estimated_naira_per_1m_equiv": 4322.0,
      "worst_word": "Make we dey go",
      "worst_word_tokens": 5,
      "worst_word_ratio": 5.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "mean_tokens_per_entry": 4.29,
      "mean_token_ratio_vs_english": 1.037,
      "cost_multiplier_vs_english": 1.059,
      "estimated_naira_per_1m_equiv": 4237.0,
      "worst_word": "Make we dey go",
      "worst_word_tokens": 4,
      "worst_word_ratio": 4.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "mean_tokens_per_entry": 4.83,
      "mean_token_ratio_vs_english": 1.077,
      "cost_multiplier_vs_english": 1.11,
      "estimated_naira_per_1m_equiv": 4441.0,
      "worst_word": "pikin",
      "worst_word_tokens": 3,
      "worst_word_ratio": 3.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "mean_tokens_per_entry": 4.55,
      "mean_token_ratio_vs_english": 1.049,
      "cost_multiplier_vs_english": 1.071,
      "estimated_naira_per_1m_equiv": 4282.0,
      "worst_word": "You don come house",
      "worst_word_tokens": 4,
      "worst_word_ratio": 2.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "mean_tokens_per_entry": 4.79,
      "mean_token_ratio_vs_english": 1.062,
      "cost_multiplier_vs_english": 1.104,
      "estimated_naira_per_1m_equiv": 4415.0,
      "worst_word": "pikin",
      "worst_word_tokens": 3,
      "worst_word_ratio": 3.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "mean_tokens_per_entry": 4.86,
      "mean_token_ratio_vs_english": 1.071,
      "cost_multiplier_vs_english": 1.087,
      "estimated_naira_per_1m_equiv": 4349.0,
      "worst_word": "pikin",
      "worst_word_tokens": 3,
      "worst_word_ratio": 3.0
    },
    {
      "language": "pcm",
      "language_name": "Pidgin",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "mean_tokens_per_entry": 4.72,
      "mean_token_ratio_vs_english": 1.009,
      "cost_multiplier_vs_english": 1.037,
      "estimated_naira_per_1m_equiv": 4149.0,
      "worst_word": "You don come house",
      "worst_word_tokens": 4,
      "worst_word_ratio": 2.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "bert_multilingual",
      "tokenizer_name": "BERT Multilingual",
      "mean_tokens_per_entry": 10.69,
      "mean_token_ratio_vs_english": 2.627,
      "cost_multiplier_vs_english": 2.391,
      "estimated_naira_per_1m_equiv": 9566.0,
      "worst_word": "pápá ọkọ̀ òfuurufú",
      "worst_word_tokens": 11,
      "worst_word_ratio": 11.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "gemma",
      "tokenizer_name": "Gemma 2",
      "mean_tokens_per_entry": 11.8,
      "mean_token_ratio_vs_english": 3.084,
      "cost_multiplier_vs_english": 2.914,
      "estimated_naira_per_1m_equiv": 11654.0,
      "worst_word": "pápá ọkọ̀ òfuurufú",
      "worst_word_tokens": 12,
      "worst_word_ratio": 12.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "gpt4_cl100k",
      "tokenizer_name": "GPT-4 (cl100k_base)",
      "mean_tokens_per_entry": 14.78,
      "mean_token_ratio_vs_english": 3.461,
      "cost_multiplier_vs_english": 3.398,
      "estimated_naira_per_1m_equiv": 13591.0,
      "worst_word": "pápá ọkọ̀ òfuurufú",
      "worst_word_tokens": 15,
      "worst_word_ratio": 15.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "gpt4o_o200k",
      "tokenizer_name": "GPT-4o (o200k_base)",
      "mean_tokens_per_entry": 10.18,
      "mean_token_ratio_vs_english": 2.516,
      "cost_multiplier_vs_english": 2.395,
      "estimated_naira_per_1m_equiv": 9581.0,
      "worst_word": "ẹ̀rọ ìbánisọ̀rọ̀",
      "worst_word_tokens": 11,
      "worst_word_ratio": 11.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "llama3",
      "tokenizer_name": "Llama 3",
      "mean_tokens_per_entry": 13.35,
      "mean_token_ratio_vs_english": 3.149,
      "cost_multiplier_vs_english": 3.076,
      "estimated_naira_per_1m_equiv": 12304.0,
      "worst_word": "pápá ọkọ̀ òfuurufú",
      "worst_word_tokens": 14,
      "worst_word_ratio": 14.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "mistral",
      "tokenizer_name": "Mistral v0.3",
      "mean_tokens_per_entry": 15.66,
      "mean_token_ratio_vs_english": 3.919,
      "cost_multiplier_vs_english": 3.503,
      "estimated_naira_per_1m_equiv": 14013.0,
      "worst_word": "ẹ̀rọ ìbánisọ̀rọ̀",
      "worst_word_tokens": 17,
      "worst_word_ratio": 17.0
    },
    {
      "language": "yor",
      "language_name": "Yorùbá",
      "tokenizer": "xlm_roberta",
      "tokenizer_name": "XLM-RoBERTa",
      "mean_tokens_per_entry": 12.85,
      "mean_token_ratio_vs_english": 3.217,
      "cost_multiplier_vs_english": 2.824,
      "estimated_naira_per_1m_equiv": 11297.0,
      "worst_word": "ẹ̀rọ ìbánisọ̀rọ̀",
      "worst_word_tokens": 14,
      "worst_word_ratio": 14.0
    }
  ],
  "top_worst_tokenized": [
    {
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "language": "Yorùbá",
      "tokenizer": "Mistral v0.3",
      "n_tokens": 17,
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "ratio": 17.0,
      "token_splits": [
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "r",
        "ọ",
        "▁",
        "ì",
        "b",
        "án",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ]
    },
    {
      "text": "pápá ọkọ̀ òfuurufú",
      "language": "Yorùbá",
      "tokenizer": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "english_equivalent": "airport",
      "english_tokens": 1,
      "ratio": 15.0,
      "token_splits": [
        "p",
        "á",
        "p",
        "á",
        " ",
        "ọ",
        "k",
        "ọ",
        "̀",
        " �",
        "�",
        "fu",
        "ur",
        "uf",
        "ú"
      ]
    },
    {
      "text": "pápá ọkọ̀ òfuurufú",
      "language": "Yorùbá",
      "tokenizer": "Mistral v0.3",
      "n_tokens": 15,
      "english_equivalent": "airport",
      "english_tokens": 1,
      "ratio": 15.0,
      "token_splits": [
        "▁p",
        "á",
        "p",
        "á",
        "▁",
        "ọ",
        "k",
        "ọ",
        "̀",
        "▁",
        "ò",
        "f",
        "uur",
        "uf",
        "ú"
      ]
    },
    {
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "language": "Yorùbá",
      "tokenizer": "GPT-4 (cl100k_base)",
      "n_tokens": 15,
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "ratio": 15.0,
      "token_splits": [
        "�",
        "�",
        "̀",
        "r",
        "ọ",
        " �",
        "�",
        "b",
        "án",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ]
    },
    {
      "text": "pápá ọkọ̀ òfuurufú",
      "language": "Yorùbá",
      "tokenizer": "Llama 3",
      "n_tokens": 14,
      "english_equivalent": "airport",
      "english_tokens": 1,
      "ratio": 14.0,
      "token_splits": [
        "p",
        "Ã¡p",
        "Ã¡",
        "Ġ",
        "á»į",
        "k",
        "á»į",
        "ÌĢ",
        "ĠÃ",
        "²",
        "fu",
        "ur",
        "uf",
        "Ãº"
      ]
    },
    {
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "language": "Yorùbá",
      "tokenizer": "Llama 3",
      "n_tokens": 14,
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "ratio": 14.0,
      "token_splits": [
        "áº¹",
        "ÌĢ",
        "r",
        "á»į",
        "ĠÃ",
        "¬",
        "b",
        "Ã¡n",
        "is",
        "á»į",
        "ÌĢ",
        "r",
        "á»į",
        "ÌĢ"
      ]
    },
    {
      "text": "ẹ̀rọ ìbánisọ̀rọ̀",
      "language": "Yorùbá",
      "tokenizer": "XLM-RoBERTa",
      "n_tokens": 14,
      "english_equivalent": "telephone",
      "english_tokens": 1,
      "ratio": 14.0,
      "token_splits": [
        "▁",
        "ẹ",
        "̀",
        "r",
        "ọ",
        "▁",
        "ì",
        "bán",
        "is",
        "ọ",
        "̀",
        "r",
        "ọ",
        "̀"
      ]
    },
    {
      "text": "pápá ọkọ̀ òfuurufú",
      "language": "Yorùbá",
      "tokenizer": "XLM-RoBERTa",
      "n_tokens": 13,
      "english_equivalent": "airport",
      "english_tokens": 1,
      "ratio": 13.0,
      "token_splits": [
        "▁pá",
        "pá",
        "▁",
        "ọ",
        "k",
        "ọ",
        "̀",
        "▁",
        "ò",
        "fu",
        "u",
        "ruf",
        "ú"
      ]
    },
    {
      "text": "ilé ẹ̀kọ́ gíga",
      "language": "Yorùbá",
      "tokenizer": "Mistral v0.3",
      "n_tokens": 13,
      "english_equivalent": "university",
      "english_tokens": 1,
      "ratio": 13.0,
      "token_splits": [
        "▁il",
        "é",
        "▁",
        "<0xE1>",
        "<0xBA>",
        "<0xB9>",
        "̀",
        "k",
        "ọ",
        "́",
        "▁g",
        "í",
        "ga"
      ]
    },
    {
      "text": "pápá ọkọ̀ òfuurufú",
      "language": "Yorùbá",
      "tokenizer": "Gemma 2",
      "n_tokens": 12,
      "english_equivalent": "airport",
      "english_tokens": 1,
      "ratio": 12.0,
      "token_splits": [
        "p",
        "áp",
        "á",
        "▁ọ",
        "k",
        "ọ",
        "̀",
        "▁ò",
        "fu",
        "ur",
        "uf",
        "ú"
      ]
    }
  ]
}