# textanalyzer/nltk_analyzer.py
import nltk
import ssl  # 新增：导入ssl模块
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist

# 新增：临时关闭SSL验证
try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
    pass
else:
    ssl._create_default_https_context = _create_unverified_https_context


# 确保必要的NLTK数据已下载
try:
    word_tokenize("test")
except LookupError:
    nltk.download('punkt')

def tokenize(text: str) -> list:
    """使用NLTK进行分词"""
    return word_tokenize(text)

def word_frequency(text: str, top_n: int = 10) -> dict:
    """返回文本中出现频率最高的n个词"""
    tokens = tokenize(text)
    # 过滤非字母字符
    words = [word.lower() for word in tokens if word.isalpha()]
    freq = FreqDist(words)
    return dict(freq.most_common(top_n))
