"""Base class of corresponding tokenizers."""

# AUTOGENERATED! DO NOT EDIT! File to edit: ../../../src/platform/tokenizer/base_tokenizer.ipynb.

# %% auto #0
__all__ = ['Vocabulary', 'VocabularyInverse', 'invert_vocabulary', 'BaseTokenizer']

# %% ../../../src/platform/tokenizer/base_tokenizer.ipynb #06272f6f-b4e3-4504-a90a-feebbf6ad821
from ...imports import *

# %% ../../../src/platform/tokenizer/base_tokenizer.ipynb #49c180c0-dd8c-4a69-b2f5-77944f6d3bcf
Vocabulary        = dict[str, int] | dict[Any, int]
VocabularyInverse = dict[int, str] | dict[int, Any]

def invert_vocabulary(vocabulary: Vocabulary) -> VocabularyInverse:
    vocabulary_inverse = {token:gate for gate, token in vocabulary.items()}
    return vocabulary_inverse  

# %% ../../../src/platform/tokenizer/base_tokenizer.ipynb #36032308-bd0e-4409-9db0-9d89fc258e5a
class BaseTokenizer(abc.ABC):

    def __init__(self, vocabulary: Vocabulary) -> None:
        self.vocabulary         = vocabulary
        self.vocabulary_inverse = invert_vocabulary(vocabulary)

    @abc.abstractmethod
    def tokenize(self, *args, **kwargs):
        raise NotImplementedError()

    @abc.abstractmethod
    def encode(self, *args, **kwargs):
        raise NotImplementedError()

    @abc.abstractmethod
    def decode(self, *args, **kwargs):
        raise NotImplementedError()
