{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NaijaML Explorer\n",
    "\n",
    "Interactive notebook to explore all NaijaML features.\n",
    "\n",
    "**Tasks covered:**\n",
    "- Task 1 & 2: Dataset loaders\n",
    "- Task 3: Nigerian constants & text preprocessing\n",
    "- Task 4: Language detection\n",
    "- Task 5: Yorùbá & Igbo diacritization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Task 1 & 2: Dataset Loaders\n",
    "\n",
    "Load Nigerian NLP datasets with a simple API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.data import load_dataset, list_datasets, dataset_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# See all available datasets\n",
    "list_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get info about a dataset (without downloading)\n",
    "dataset_info(\"naijasenti\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NaijaSenti (Sentiment Analysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Yorùbá sentiment data\n",
    "yor_sentiment = load_dataset(\"naijasenti\", lang=\"yor\", split=\"train\")\n",
    "print(f\"Loaded {len(yor_sentiment)} Yorùbá samples\")\n",
    "yor_sentiment[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try other languages: hau, ibo, pcm\n",
    "# YOUR EXPLORATION HERE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MasakhaNER (Named Entity Recognition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Hausa NER data\n",
    "hau_ner = load_dataset(\"masakhaner\", lang=\"hau\", split=\"test\")\n",
    "print(f\"Loaded {len(hau_ner)} Hausa NER samples\")\n",
    "hau_ner[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize NER tags\n",
    "sample = hau_ner[0]\n",
    "for token, tag in zip(sample[\"tokens\"], sample[\"ner_tags\"]):\n",
    "    if tag != \"O\":\n",
    "        print(f\"{token:20} → {tag}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MasakhaNEWS (News Classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Hausa news data\n",
    "hau_news = load_dataset(\"masakhanews\", lang=\"hau\", split=\"test\")\n",
    "print(f\"Loaded {len(hau_news)} Hausa news articles\")\n",
    "hau_news[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count by category\n",
    "from collections import Counter\n",
    "Counter(item[\"label\"] for item in hau_news)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Task 3: Nigerian Constants & Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nigerian Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.utils import (\n",
    "    STATES, STATE_NAMES, LGAS, BANKS, TELCOS,\n",
    "    format_naira, parse_naira,\n",
    "    is_valid_phone, normalize_phone, get_telco,\n",
    "    is_valid_bvn, is_valid_nin\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nigerian states and capitals\n",
    "print(f\"{len(STATES)} states + FCT\")\n",
    "print(f\"Lagos capital: {STATES['Lagos']}\")\n",
    "print(f\"Kano capital: {STATES['Kano']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LGAs\n",
    "print(\"Lagos LGAs:\", LGAS[\"Lagos\"][:5], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nigerian banks\n",
    "print(\"Banks:\", list(BANKS.keys())[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Telcos and their prefixes\n",
    "for telco, info in TELCOS.items():\n",
    "    print(f\"{telco}: {info['prefixes'][:3]}...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naira Formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format amounts\n",
    "print(format_naira(1500000))\n",
    "print(format_naira(50000, include_kobo=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse amounts\n",
    "print(parse_naira(\"₦1,500,000.00\"))\n",
    "print(parse_naira(\"NGN 50,000\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Phone Number Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Validate phone numbers\n",
    "test_phones = [\"08031234567\", \"+2348012345678\", \"12345\", \"09012345678\"]\n",
    "for phone in test_phones:\n",
    "    valid = is_valid_phone(phone)\n",
    "    telco = get_telco(phone) if valid else None\n",
    "    print(f\"{phone:20} valid={valid:5} telco={telco}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize to international format\n",
    "print(normalize_phone(\"08031234567\"))\n",
    "print(normalize_phone(\"0803-123-4567\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.nlp import (\n",
    "    clean_nigerian_text, clean_social_media,\n",
    "    mask_pii, find_phones, find_naira_amounts,\n",
    "    normalize_unicode, strip_diacritics,\n",
    "    extract_hashtags, extract_mentions\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean social media text\n",
    "tweet = \"@user Check https://t.co/abc This film too sweet!!! #Nollywood #NaijaFilm\"\n",
    "print(\"Original:\", tweet)\n",
    "print(\"Cleaned: \", clean_social_media(tweet))\n",
    "print(\"Hashtags:\", extract_hashtags(tweet))\n",
    "print(\"Mentions:\", extract_mentions(tweet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mask PII (personally identifiable information)\n",
    "text_with_pii = \"Call me on 08012345678 or email test@example.com for the ₦50,000 deal\"\n",
    "print(\"Original:\", text_with_pii)\n",
    "print(\"Masked:  \", mask_pii(text_with_pii))\n",
    "print(\"Phones:  \", find_phones(text_with_pii))\n",
    "print(\"Amounts: \", find_naira_amounts(text_with_pii))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Yorùbá diacritic handling\n",
    "yoruba_text = \"Ọjọ́ dára púpọ̀, ẹ kú iṣẹ́\"\n",
    "print(\"Original:        \", yoruba_text)\n",
    "print(\"Strip diacritics:\", strip_diacritics(yoruba_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All-in-one cleaning\n",
    "messy_text = \"@someone Check this ₦100k deal!!! https://bit.ly/xyz Call 08012345678 #Lagos\"\n",
    "print(\"Original:\", messy_text)\n",
    "print(\"Cleaned: \", clean_nigerian_text(messy_text, mask_pii_data=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Task 4: Language Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.nlp import (\n",
    "    detect_language,\n",
    "    detect_language_with_confidence,\n",
    "    detect_all_languages,\n",
    "    SUPPORTED_LANGUAGES\n",
    ")\n",
    "\n",
    "print(\"Supported languages:\", SUPPORTED_LANGUAGES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic detection\n",
    "samples = [\n",
    "    \"Ọjọ́ dára púpọ̀, ẹ kú iṣẹ́\",        # Yorùbá\n",
    "    \"Ina kwana, yaya aiki?\",             # Hausa\n",
    "    \"Kedu ka ị mere? Ọ dị mma\",          # Igbo\n",
    "    \"Wetin dey happen for this country?\", # Pidgin\n",
    "    \"The weather is quite pleasant today\", # English\n",
    "]\n",
    "\n",
    "for text in samples:\n",
    "    lang = detect_language(text)\n",
    "    print(f\"{lang}: {text[:40]}...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Detection with confidence\n",
    "for text in samples:\n",
    "    lang, conf = detect_language_with_confidence(text)\n",
    "    print(f\"{lang} ({conf:5.1%}): {text[:35]}...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all language probabilities\n",
    "text = \"Wetin dey happen?\"\n",
    "scores = detect_all_languages(text)\n",
    "print(f\"Text: '{text}'\\n\")\n",
    "for lang, prob in sorted(scores.items(), key=lambda x: -x[1]):\n",
    "    bar = \"█\" * int(prob * 30)\n",
    "    print(f\"{lang}: {bar} {prob:.1%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# YOUR EXPLORATION: Try your own text!\n",
    "my_text = \"\"  # <-- put your text here\n",
    "\n",
    "if my_text:\n",
    "    lang, conf = detect_language_with_confidence(my_text)\n",
    "    print(f\"Detected: {lang} ({conf:.1%})\")\n",
    "    print(\"\\nAll scores:\")\n",
    "    for l, p in sorted(detect_all_languages(my_text).items(), key=lambda x: -x[1]):\n",
    "        print(f\"  {l}: {p:.1%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Task 5: Diacritization\n",
    "\n",
    "Restore diacritics to plain Yorùbá and Igbo text. Yorùbá uses tonal marks (á, à) and dot-below characters (ọ, ẹ, ṣ) that are often omitted when typing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Yorùbá Full Diacritization\n",
    "\n",
    "Restores both tonal marks (á, à) and dot-below (ọ, ẹ, ṣ). Uses **bigram word context** to disambiguate — e.g. the word \"ile\" becomes \"ilé\" (house) after \"inu\" but \"ilẹ̀\" (ground) after \"ori\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.nlp import diacritize\n",
    "\n",
    "# Everyday sentences\n",
    "sentences = [\n",
    "    \"Ojo dara pupo\",\n",
    "    \"E ku ise o, bawo ni?\",\n",
    "    \"Mo fe ra onje ni oja\",\n",
    "    \"Awon omo de n sere ni ona\",\n",
    "    \"Oluwa a bukun fun e\",\n",
    "]\n",
    "\n",
    "for s in sentences:\n",
    "    print(f\"{s:35s} \\u2192 {diacritize(s)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bigram context disambiguation\n",
    "# The same word \"ile\" gets different diacritics based on context\n",
    "pairs = [\n",
    "    (\"inu ile\",  \"'inú ilé' = inside the house\"),\n",
    "    (\"ori ile\",  \"'orí ilẹ̀' = on the ground\"),\n",
    "    (\"ile nla\",  \"no context → unigram default\"),\n",
    "]\n",
    "\n",
    "for text, note in pairs:\n",
    "    result = diacritize(text)\n",
    "    print(f\"{text:15s} → {result:15s}  ({note})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Yorùbá Dot-Below Only\n",
    "\n",
    "A simpler, higher-accuracy mode that only restores ọ, ẹ, ṣ (no tonal marks). Use when you need reliable results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.nlp import diacritize_dot_below\n",
    "\n",
    "sentences = [\n",
    "    \"Ojo dara pupo\",\n",
    "    \"E ku ise o\",\n",
    "    \"Omo mi dara\",\n",
    "]\n",
    "\n",
    "for s in sentences:\n",
    "    full = diacritize(s)\n",
    "    dots = diacritize_dot_below(s)\n",
    "    print(f\"{s:20s} \\u2192 full: {full:25s} dot-below: {dots}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Igbo Diacritization\n",
    "\n",
    "Restores dot-below vowels (ị, ọ, ụ) in Igbo text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.nlp import diacritize_igbo\n",
    "\n",
    "igbo_sentences = [\n",
    "    \"Kedu ka i mere\",\n",
    "    \"O di mma\",\n",
    "    \"Unu nile ekele\",\n",
    "    \"Chineke gozie gi\",\n",
    "]\n",
    "\n",
    "for s in igbo_sentences:\n",
    "    print(f\"{s:25s} \\u2192 {diacritize_igbo(s)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Strip Diacritics\n",
    "\n",
    "Go the other direction — remove diacritics for search, indexing, or comparison."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from naijaml.nlp.diacritizer import strip_diacritics\n",
    "\n",
    "text = \"Ọjọ́ dára púpọ̀, ẹ kú iṣẹ́\"\n",
    "print(f\"Original:          {text}\")\n",
    "print(f\"Strip all:         {strip_diacritics(text)}\")\n",
    "print(f\"Strip tones only:  {strip_diacritics(text, tones_only=True)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try your own text!\n",
    "my_text = \"\"  # <-- type undiacritized Yorùbá here\n",
    "\n",
    "if my_text:\n",
    "    print(f\"Full:      {diacritize(my_text)}\")\n",
    "    print(f\"Dot-below: {diacritize_dot_below(my_text)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Your Playground\n",
    "\n",
    "Use the cells below for your own experiments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Your code here\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "naijaml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
