from typing import List, Optional

class PyTokenizer:
    def encode(self, text: str) -> List[int]: ...
    def decode(self, tokens: List[int]) -> str: ...
    def count(self, text: str) -> int: ...
    def count_till_limit(self, text: str, token_limit: int) -> Optional[int]: ...
    def pretokenize(self, text: str) -> List[str]: ...

def tokenizer_for_model(model_name: str) -> PyTokenizer: ...
def tokenizer_for_encoding(encoding_name: str) -> PyTokenizer: ...
def supported_models() -> List[str]: ...
def supported_encodings() -> List[str]: ...

RUST_BACKEND_VERSION: str
PYTHON_API_VERSION: str
