import os
import logging
from typing import Dict, Optional, OrderedDict, Sequence, Tuple, Union
from .configuration_auto import replace_list_option_in_docstrings
from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.configuration_utils import PretrainedConfig

logger: logging.Logger
TOKENIZER_MAPPING: OrderedDict[PretrainedConfig, Tuple[PreTrainedTokenizer, PreTrainedTokenizerFast]] = ...
NO_CONFIG_TOKENIZER: Sequence[Union[PreTrainedTokenizer, PreTrainedTokenizerFast]] = ...
SLOW_TOKENIZER_MAPPING: Dict[PretrainedConfig, Union[PreTrainedTokenizer, PreTrainedTokenizerFast]] = ...

def tokenizer_class_from_name(class_name: str) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: ...

def get_tokenizer_config(pretrained_model_name_or_path: Union[str, os.PathLike], cache_dir: Optional[Union[str, os.PathLike]] = ..., force_download: bool = ..., resume_download: bool = ..., proxies: Optional[Dict[str, str]] = ..., use_auth_token: Optional[Union[bool, str]] = ..., revision: Optional[str] = ..., local_files_only: bool = ..., **kwargs) -> Dict: ...

class AutoTokenizer:
    def __init__(self) -> None: ...
    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike] = ..., *inputs, **kwargs) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: ...
