from __future__ import annotations

import re

import jieba_next

try:
    from whoosh.analysis import (
        LowercaseFilter,
        RegexAnalyzer,
        StemFilter,
        StopFilter,
        Token,
        Tokenizer,
    )
    from whoosh.lang.porter import stem
except Exception:  # whoosh missing
    Token = object
    stem = lambda x: x  # noqa: E731

    class Tokenizer:
        pass


STOP_WORDS = frozenset((
    "a",
    "an",
    "and",
    "are",
    "as",
    "at",
    "be",
    "by",
    "can",
    "for",
    "from",
    "have",
    "if",
    "in",
    "is",
    "it",
    "may",
    "not",
    "of",
    "on",
    "or",
    "tbd",
    "that",
    "the",
    "this",
    "to",
    "us",
    "we",
    "when",
    "will",
    "with",
    "yet",
    "you",
    "your",
    "的",
    "了",
    "和",
))

accepted_chars = re.compile(r"[\u4E00-\u9FD5]+")


class ChineseTokenizer(Tokenizer):
    def __call__(self, text, **kargs):
        words = jieba_next.tokenize(text, mode="search")
        token = Token()
        for w, start_pos, stop_pos in words:
            if not accepted_chars.match(w) and len(w) <= 1:
                continue
            token.original = token.text = w
            token.pos = start_pos
            token.startchar = start_pos
            token.endchar = stop_pos
            yield token


def ChineseAnalyzer(stoplist=STOP_WORDS, minsize=1, stemfn=stem, cachesize=50000):
    return (
        ChineseTokenizer()
        | LowercaseFilter()
        | StopFilter(stoplist=stoplist, minsize=minsize)
        | StemFilter(stemfn=stemfn, ignore=None, cachesize=cachesize)
    )
