from typing import List, Optional
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import nltk

nltk.download('punkt_tab')


# Ensure necessary data is downloaded
try:
    nltk.data.find("tokenizers/punkt")
except LookupError:
    nltk.download("punkt")
try:
    nltk.data.find("corpora/stopwords")
except LookupError:
    nltk.download("stopwords")


def tokenize(text: str) -> List[str]:
    return word_tokenize(text)


def tokenize_lower(text: str) -> List[str]:
    return [tok.lower() for tok in word_tokenize(text)]


def remove_stopwords(tokens: List[str], lang: str = "english") -> List[str]:
    stop = set(stopwords.words(lang))
    return [tok for tok in tokens if tok.lower() not in stop]


def preprocess(text: str, lang: str = "english") -> List[str]:
    toks = tokenize_lower(text)
    toks = remove_stopwords(toks, lang=lang)
    return toks
