# AUTOGENERATED! DO NOT EDIT! File to edit: 01_tokenizer.ipynb (unless otherwise specified).

__all__ = ['Tokenizer', 'SoMaJoTokenizer']

# Cell
from abc import ABC, abstractmethod

# Cell
class Tokenizer(ABC):
    @abstractmethod
    def split(self, text):
        """`split` returns an iterable over sentences. Every sentence is an iterable over string tokens."""
        pass

# Cell
class SoMaJoTokenizer(Tokenizer):
    def __init__(self, language):
        from somajo import SoMaJo

        tokenizer_type = {"de": "de_CMC", "en": "en_PTB"}[language]
        self.tokenizer = SoMaJo(
            tokenizer_type, split_camel_case=True, split_sentences=True
        )

    def split(self, text):
        return self.tokenizer.tokenize_text([text])