"""
Unified parameter definitions for the Kin-Tokenizer.

All tunable constants, regex patterns, and configuration defaults live here.
Import from this module instead of hardcoding values across the codebase.
"""

import os

# ───────────────────────────── Version ─────────────────────────────
# Single source of truth — version.py re-exports this value.
VERSION = "3.1.0"

# ───────────────────────────── Tokenizer Defaults ──────────────────
DEFAULT_VOCAB_SIZE = 50_251
"""Default target vocabulary size (256 byte tokens + PAD + merges + 6 special)."""

MIN_VOCAB_SIZE = 257
"""Absolute minimum: 256 byte tokens + PAD."""

DEFAULT_MIN_FREQUENCY = 2
"""Minimum pair frequency to continue BPE merges.

For Kinyarwanda (an agglutinative Bantu language), a low threshold is critical
to capture rare but morphologically productive verb forms like
n-a-ka-gur-ir-anya (I repeatedly bought for).
"""

# ───────────────────────────── Special Tokens ──────────────────────
SPECIAL_TOKENS = (
    "<|EOS|>",
    "<|BOS|>",
    "<|SEP|>",
    "<|MASK|>",
    "<|UNK|>",
    "<|CLS|>",
)

PAD_TOKEN = "<|PAD|>"
PAD_TOKEN_ID = 0

# ───────────────────────────── Space Strategy ──────────────────────
METASPACE = "\u2581"  # ▁  SentencePiece metaspace (U+2581)

SPACE_STRATEGY_METASPACE = "metaspace"
SPACE_STRATEGY_GPT2 = "gpt2"
DEFAULT_SPACE_STRATEGY = SPACE_STRATEGY_METASPACE

# ───────────────────────────── Regex Patterns ──────────────────────
# Updated from tiktoken/o200k_base insights:
#  - \p{M} (combining marks) appended to every \p{L} group so accented
#    Kinyarwanda characters and French loanword diacritics stay together.
#  - \p{N}+ (unlimited digits) replaces {1,3} to avoid splitting numbers.
#
# Metaspace pattern (new training):
REGEX_PATTERN_METASPACE = (
    r"""\u2581'[\p{L}\p{M}]+"""    # ▁'word  (apostrophe-start with metaspace)
    r"""|'[\p{L}\p{M}]+"""         # 'word   (bare apostrophe contraction)
    r"""|\u2581[\p{L}\p{M}]+"""    # ▁word   (standard metaspace-prefixed word)
    r"""|[\p{L}\p{M}]+"""          # bare word (after symbol split, e.g. hello_world → _ + world)
    r"""|\u2581?\p{N}+"""          # ▁123 or 123  (numbers, unlimited digits)
    r"""|\u2581?[^\s\u2581\p{L}\p{N}]+"""  # ▁punct or punct
    r"""|\s+"""                    # whitespace
)

# GPT-2 compat pattern (old checkpoints):
REGEX_PATTERN_GPT2 = (
    r"""'[\p{L}\p{M}]+"""         # 'word
    r"""| ?[\p{L}\p{M}]+"""       # optional-space + word
    r"""| ?\p{N}+"""              # optional-space + number (unlimited digits)
    r"""| ?[^\s\p{L}\p{N}]+"""    # optional-space + punctuation
    r"""|\s+"""                    # whitespace
)

# ───────────────────────────── Training Defaults ───────────────────
# ╔══════════════════════════════════════════════════════════════════╗
# ║  Tuned for: ~880 MB Kinyarwanda corpus                         ║
# ║             14 CPU cores  ·  24 GB RAM                         ║
# ║                                                                ║
# ║  Corpus stats (estimated):                                     ║
# ║    ~880 MB UTF-8 → ~150M words → ~900M byte-tokens             ║
# ║    ~8.8M lines at ~100 bytes/line average                      ║
# ║                                                                ║
# ║  Memory budget:                                                ║
# ║    tokens_chunks (u32 per byte) ≈ 900M × 4B = 3.6 GB          ║
# ║    pair freq FxHashMap           ≈ 1–2 GB                      ║
# ║    Python overhead + OS          ≈ 4–6 GB                      ║
# ║    Headroom for checkpoint I/O   ≈ remaining (~12 GB)          ║
# ╚══════════════════════════════════════════════════════════════════╝

DEFAULT_NUM_PROCESSES = 14
"""Default worker count for Python multiprocessing / Rayon threads.
Matches the 14 available CPU cores."""

LARGE_CORPUS_CPU_CAP = 14
"""Maximum process count for the Python-fallback train() path on large corpora.
Previously hardcoded to 8; raised to use all 14 cores given 24 GB RAM."""

DEFAULT_STREAM_LINES = 200_000
"""Lines per Rust streaming batch for file-based training.

880 MB at ~100 bytes/line ≈ 8.8M lines → 44 batches at 200K.
Each batch is pretokenised in parallel across 14 Rayon threads.
Larger batches reduce I/O round-trips and Rayon thread-pool startup cost.
Peak memory per batch: ~200K lines × ~100B ≈ 20 MB (negligible)."""

RUST_MERGE_BATCH_SIZE = 5_000
"""Number of BPE merges per Rust call.

With incremental pair-count updates in the Rust trainer, each batch runs
entirely in Rust without Python↔Rust serialisation of the full corpus.
5 000 merges per call → ~10 calls for a 50K-merge run.
Memory: the Rust trainer keeps one corpus copy (~3.6 GB) plus an
incremental pair-frequency map (~1–2 GB) — fits well within 24 GB."""

CHECKPOINT_EVERY_N_MERGES = 5_000
"""Save tokenizer + token chunks every N merges during streamed training.

Aligned with RUST_MERGE_BATCH_SIZE so every batch boundary is a checkpoint.
Pickle-dumping the ~3.6 GB token chunks takes 20–40 s on NVMe;
checkpointing every 5 000 merges (~10 writes total) balances safety
vs. throughput for a multi-hour training run."""

PYTHON_FALLBACK_CHECKPOINT_EVERY = 100
"""Save interval for the pure-Python merge loop (no Rust).

Python merges are much slower (~1–5 s each on 880 MB), so checkpointing
every 100 merges avoids excessive I/O while still limiting re-work on crash."""

MMAP_THRESHOLD_BYTES = 128 * 1024 * 1024  # 128 MB
"""Use mmap for reading datasets larger than this."""

# ───────────────────────────── Multiprocessing ─────────────────────
DECODE_MULTIPROCESS_THRESHOLD = 5_000
"""Minimum token count before falling back to multiprocess decode."""

# ───────────────────────────── Kinyarwanda Morphology ──────────────
# Common Bantu noun class prefixes and verbal subject markers.
# BPE naturally discovers these; this list is for morphological analysis/evaluation.
KINYARWANDA_PREFIXES = [
    # Noun class prefixes  (singular / plural)
    "umu", "aba",   # class 1/2  (persons)
    "imi",          # class 3/4  (trees, plants) — umu already listed above
    "iki", "ibi",   # class 5/6  (augmentatives; also icy-)
    "i",   "ama",   # class 7/8  (liquids, collectives)
    "in",           # class 9/10 (animals, loanwords)
    "uru",          # class 11   (long/thin objects)
    "aka", "utu",   # class 12/13 (diminutives)
    "ubu",          # class 14   (abstract nouns)
    "uku",          # class 15   (infinitives)
    "aha",          # class 16   (locatives)
    # Subject markers
    "n", "u", "a", "tu", "mu", "ba",
    # Tense / aspect markers
    "ra", "ri", "za", "ki",
    # Apostrophe-bound prefixes (handled by regex)
    "b'", "n'", "w'", "y'", "k'", "m'",
]

# ───────────────────────────── Loan Word Filtering ─────────────────
# French/English loan words commonly used in Kinyarwanda that should NOT
# be stripped by the non-Kinyarwanda word filter.
LOAN_WORD_ALLOWLIST = frozenset({
    # French-origin (widely adopted in Kinyarwanda)
    "telephone", "bureau", "police", "radio", "hotel", "garage",
    "journal", "president", "restaurant", "cinema", "programme",
    "banque", "classe", "ministre", "parlement", "universite",
    "boutique", "pharmacie", "hopital", "ecole", "professeur",
    "directeur", "television", "ordinateur",
    # English-origin
    "football", "computer", "internet", "email", "basketball",
    "volleyball", "bus", "taxi",
    # Technology / modern life
    "telefoni", "mudasobwa", "interineti",
    # Education (Kinyarwandised borrowings)
    "komite", "raporo", "dosiye", "budije", "perezida",
})

# ───────────────────────────── Dataset / Sequence Defaults ─────────

DEFAULT_SEQUENCE_LENGTH = 512
"""Token-level context window per training sequence.

512 is the BERT-era standard and works well for an 880 MB Kinyarwanda
corpus on a single GPU with 8-16 GB VRAM.  Fits ~3-5 paragraphs of
agglutinative Kinyarwanda text per window."""

DEFAULT_STEP_SIZE = None
"""Sliding-window stride. None → sequence_length // 2 (50% overlap).

50% overlap doubles the number of training sequences compared to no
overlap, which is important for a low-resource corpus. It also means
every token (except the first and last windows) appears in at least
two training examples with different surrounding context — acting as
a form of data augmentation."""

DEFAULT_EVAL_RATIO = 0.15
"""Fraction of the text (by paragraph) held out for evaluation.

Split is done at paragraph level (double-newline boundaries) BEFORE
encoding to prevent any data leakage between train and eval sequences.
15% eval leaves 85% for training while keeping a sufficiently large,
held-out evaluation set for local language-model pretraining."""

DEFAULT_EVAL_SEED = 42
"""Random seed for the train/eval paragraph shuffle.

Fixed seed ensures deterministic, reproducible splits across runs."""

# ── Pre-configured profiles ──
SEQUENCE_PROFILES = {
    "small": {
        "sequence_length": 256,
        "step_size": 128,
        "description": "CPU / Mac M-series / <8 GB VRAM — fast iteration, ~3.5M sequences from 880 MB",
    },
    "medium": {
        "sequence_length": 512,
        "step_size": 256,
        "description": "Single GPU 8-16 GB VRAM — balanced (recommended default), ~1.75M sequences from 880 MB",
    },
    "large": {
        "sequence_length": 1024,
        "step_size": 512,
        "description": "Single GPU ≥24 GB VRAM or multi-GPU — longer context, ~875K sequences from 880 MB",
    },
}

# ───────────────────────────── Rust Preprocessing Patterns ─────────
# These are the regex patterns compiled in merger/src/lib.rs.
# Documented here for cross-reference — actual compilation is in Rust.
RUST_RE_NEWLINES = r"\n{3,}"
RUST_RE_TRAIL = r"[^\S\n]+\n"
RUST_RE_SPACES = r"[^\S\r\n]+"
RUST_RE_PUNCT = r"([\p{L}\p{M}])\s+([.,;:!?])"
RUST_RE_URL = r"https?://\S+|www\.\S+"
RUST_RE_TOKEN_PATTERN = (
    r"▁'[\p{L}\p{M}]+|'[\p{L}\p{M}]+|▁[\p{L}\p{M}]+|▁?\p{N}+|▁?[^\s▁\p{L}\p{N}]+|\s+"
)

# ───────────────────────────── File I/O ────────────────────────────
RUST_FILE_BUF_SIZE = 256 * 1024  # 256 KB — documented for lib.rs cross-ref

# ───────────────────────────── Email / Monitoring (env-based) ──────
SMTP_HOST = os.environ.get("SMTP_HOST", "smtp.gmail.com")
SMTP_PORT = int(os.environ.get("SMTP_PORT", "465"))
EMAIL_ID = os.environ.get("EMAIL_ID", "")
EMAIL_PASSWORD = os.environ.get("EMAIL_PASSWORD", "")
WANDB_KEY = os.environ.get("WANDB_API_KEY", "")
