orangecontrib.text.tag.pos — Orange3 Text Mining documentation

# Source code for orangecontrib.text.tag.pos

    from typing import List, Callable

    import nltk
    import numpy as np

    from Orange.util import wrap_callback, dummy_callback

    from orangecontrib.text import Corpus
    from orangecontrib.text.misc import wait_nltk_data
    from orangecontrib.text.preprocess import TokenizedPreprocessor
    from orangecontrib.text.util import chunkable

    __all__ = ['POSTagger', 'StanfordPOSTagger', 'AveragedPerceptronTagger', 'MaxEntTagger']

    class POSTagger(TokenizedPreprocessor):
        """A class that wraps `nltk.TaggerI` and performs Corpus tagging. """
        def __init__(self, tagger):
            self.tagger = tagger.tag_sents

        def __call__(self, corpus: Corpus, callback: Callable = None,
                     **kw) -> Corpus:
            """ Marks tokens of a corpus with POS tags. """
            if callback is None:
                callback = dummy_callback
            corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))

            assert corpus.has_tokens()
            callback(0.2, "POS Tagging...")
            tags = np.array(self._preprocess(corpus.tokens, **kw), dtype=object)
            corpus.pos_tags = tags
            return corpus

        @chunkable
        def _preprocess(self, tokens: List[List[str]]) -> List[List[str]]:
            return list(map(lambda sent: list(map(lambda x: x[1], sent)),
                            self.tagger(tokens)))

    class StanfordPOSTaggerError(Exception):
        pass

    class StanfordPOSTagger(nltk.StanfordPOSTagger, POSTagger):
        name = 'Stanford POS Tagger'

        @classmethod
        def check(cls, path_to_model, path_to_jar):
            """ Checks whether provided `path_to_model` and `path_to_jar` are valid.

            Raises:
                ValueError: in case at least one of the paths is invalid.

            Notes:
                Can raise an exception if Java Development Kit is not installed or not properly configured.

            Examples:
                >>> try:
                ...    StanfordPOSTagger.check('path/to/model', 'path/to/stanford.jar')
                ... except ValueError as e:
                ...    print(e)
                Could not find stanford-postagger.jar jar file at path/to/stanford.jar

            """
            try:
                cls(path_to_model, path_to_jar).tag(())
            except OSError as e:
                raise StanfordPOSTaggerError(
                    'Either Java SDK not installed or some of the '
                    'files are invalid.\n' + str(e))
            except LookupError as e:
                raise StanfordPOSTaggerError(str(e).strip(' =\n'))

    class AveragedPerceptronTagger(POSTagger):
        name = 'Averaged Perceptron Tagger'

        @wait_nltk_data
        def __init__(self):
            super().__init__(nltk.PerceptronTagger())

    class MaxEntTagger(POSTagger):
        name = 'Treebank POS Tagger (MaxEnt)'

        @wait_nltk_data
        def __init__(self):
            tagger = nltk.data.load('taggers/maxent_treebank_pos_tagger/english.pickle')
            super().__init__(tagger)