orangecontrib.text.vectorization.bagofwords — Orange3 Text Mining documentation

# Source code for orangecontrib.text.vectorization.bagofwords

    """ This module constructs a new corpus with tokens as features.

    First create a corpus::

        >>> from orangecontrib.text import Corpus
        >>> corpus = Corpus.from_file('deerwester')
        >>> corpus.domain
        [ | Category] {Text}

    Then create :class:`BowVectorizer` object and call transform:

        >>> from orangecontrib.text.vectorization.bagofwords import BowVectorizer
        >>> bow = BowVectorizer()
        >>> new_corpus = bow.transform(corpus)
        >>> new_corpus.domain
        [a, abc, and, applications, binary, computer, engineering, eps, error, for,
        generation, graph, human, in, interface, intersection, iv, lab, machine,
        management, measurement, minors, of, opinion, ordering, paths, perceived,
        quasi, random, relation, response, survey, system, testing, the, time, to,
        trees, unordered, user, well, widths | Category] {Text}

    """

    from collections import OrderedDict
    from functools import partial

    import numpy as np
    from Orange.util import dummy_callback
    from gensim import corpora, models, matutils
    from sklearn.preprocessing import normalize

    from orangecontrib.text.vectorization.base import BaseVectorizer,\
        SharedTransform, VectorizationComputeValue

    class BowVectorizer(BaseVectorizer):
        name = 'BoW Vectorizer'

        COUNT = 'Count'
        BINARY = 'Binary'
        SUBLINEAR = 'Sublinear'
        NONE = '(None)'
        IDF = 'IDF'
        SMOOTH = 'Smooth IDF'
        L1 = 'L1 (Sum of elements)'
        L2 = 'L2 (Euclidean)'

        wlocals = OrderedDict((
            (COUNT, lambda tf: tf),
            (BINARY, lambda tf: np.greater(tf, 0).astype(int) if tf.size
                                else np.array([], dtype=int)),
            (SUBLINEAR, lambda tf: 1 + np.log(tf)),
        ))

        wglobals = OrderedDict((
            (NONE, lambda df, N: 1),
            (IDF, lambda df, N: np.log(N/df)),
            (SMOOTH, lambda df, N: np.log(1 + N/df)),
        ))

        norms = OrderedDict((
            (NONE, None),
            (L1, partial(normalize, norm='l1')),
            (L2, partial(normalize, norm='l2')),
        ))

        def __init__(self, norm=NONE, wlocal=COUNT, wglobal=NONE):
            self.norm = norm
            self.wlocal = wlocal
            self.wglobal = wglobal

        def _transform(self, corpus, source_dict=None, callback=dummy_callback):
            if not (len(corpus.dictionary) or source_dict) or not len(corpus):
                return corpus
            temp_corpus = list(corpus.ngrams_iterator(' ', include_postags=True))
            dic = corpora.Dictionary(temp_corpus, prune_at=None) if not source_dict else source_dict
            callback(0.3)
            temp_corpus = [dic.doc2bow(doc) for doc in temp_corpus]
            model = models.TfidfModel(dictionary=dic, normalize=False,
                                      wlocal=self.wlocals[self.wlocal],
                                      wglobal=self.wglobals[self.wglobal])
            callback(0.6)

            X = matutils.corpus2csc(model[temp_corpus], dtype=float, num_terms=len(dic)).T
            norm = self.norms[self.norm]
            if norm:
                X = norm(X)
            callback(0.9)

            # set compute values
            shared_cv = SharedTransform(self, corpus.used_preprocessor,
                                        source_dict=dic)
            cv = [VectorizationComputeValue(shared_cv, dic[i])
                  for i in range(len(dic))]

            corpus = self.add_features(corpus, X, dic, cv, var_attrs={'bow-feature': True})
            callback(1)
            return corpus

        def report(self):
            return (('Term Frequency', self.wlocal),
                    ('Document Frequency', self.wglobal),
                    ('Regularization', self.norm),)