class Tokenizer:
    """
    An abstract class representing the tokenizer interfact.
    """
    def tokenize(self, text):
        """
        Extend this class for a custom tokenizer.
        """
        raise NotImplementedError()
