"""
Corpus data module for Afaan Oromoo NLP.

Provides access to built-in corpus data like stopwords and common words.
"""

import os
from typing import List, Set

# Directory of this file
_CORPUS_DIR = os.path.dirname(__file__)


def _load_corpus_file(filename: str) -> List[str]:
    filepath = os.path.join(_CORPUS_DIR, filename)
    if not os.path.exists(filepath):
        return []

    with open(filepath, "r", encoding="utf-8") as f:
        return [line.strip() for line in f if line.strip()]


def load_stopwords() -> Set[str]:
    return set(_load_corpus_file("stopwords.txt"))


def load_common_words(limit: int = 1000) -> List[str]:
    words = _load_corpus_file("common_words.txt")
    return words[:limit]


# Cached accessors
def get_stopwords() -> Set[str]:
    if not hasattr(get_stopwords, "_cache"):
        get_stopwords._cache = load_stopwords()
    return get_stopwords._cache


def get_common_words(limit: int = 1000) -> List[str]:
    cache = getattr(get_common_words, "_cache", {})
    if limit not in cache:
        cache[limit] = load_common_words(limit)
        get_common_words._cache = cache
    return cache[limit]


__all__ = [
    "load_stopwords",
    "load_common_words",
    "get_stopwords",
    "get_common_words",
]
