import regex as re
from transformers import AutoTokenizer

class SindhiTokenizer:
    def __init__(self, model_path="aakashMeghwar01/SindhiLM-Tokenizer-v1"):
        try:
            self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
        except:
            self.tokenizer = None
        self.pattern = r"[\u0600-\u06FF]+|[^\u0600-\u06FF\s]+"

    def tokenize(self, text):
        if not text: return []
        return re.findall(self.pattern, text)

# This is the alias that fixes your reputation and the ValueError
TokenizersBackend = SindhiTokenizer
