nltk.classify.maxent — textblob-de 0.4.2 documentation

# Source code for nltk.classify.maxent

    # Natural Language Toolkit: Maximum Entropy Classifiers
    #
    # Copyright (C) 2001-2015 NLTK Project
    # Author: Edward Loper <edloper@gmail.com>
    #         Dmitry Chichkov <dchichkov@gmail.com> (TypedMaxentFeatureEncoding)
    # URL: <http://nltk.org/>
    # For license information, see LICENSE.TXT

    """
    A classifier model based on maximum entropy modeling framework.  This
    framework considers all of the probability distributions that are
    empirically consistent with the training data; and chooses the
    distribution with the highest entropy.  A probability distribution is
    "empirically consistent" with a set of training data if its estimated
    frequency with which a class and a feature vector value co-occur is
    equal to the actual frequency in the data.

    Terminology: 'feature'
    ======================
    The term *feature* is usually used to refer to some property of an
    unlabeled token.  For example, when performing word sense
    disambiguation, we might define a ``'prevword'`` feature whose value is
    the word preceding the target word.  However, in the context of
    maxent modeling, the term *feature* is typically used to refer to a
    property of a "labeled" token.  In order to prevent confusion, we
    will introduce two distinct terms to disambiguate these two different
    concepts:

      - An "input-feature" is a property of an unlabeled token.
      - A "joint-feature" is a property of a labeled token.

    In the rest of the ``nltk.classify`` module, the term "features" is
    used to refer to what we will call "input-features" in this module.

    In literature that describes and discusses maximum entropy models,
    input-features are typically called "contexts", and joint-features
    are simply referred to as "features".

    Converting Input-Features to Joint-Features
    -------------------------------------------
    In maximum entropy models, joint-features are required to have numeric
    values.  Typically, each input-feature ``input_feat`` is mapped to a
    set of joint-features of the form:

    |   joint_feat(token, label) = { 1 if input_feat(token) == feat_val
    |                              {      and label == some_label
    |                              {
    |                              { 0 otherwise

    For all values of ``feat_val`` and ``some_label``.  This mapping is
    performed by classes that implement the ``MaxentFeatureEncodingI``
    interface.
    """
    from __future__ import print_function, unicode_literals
    __docformat__ = 'epytext en'

    try:
        import numpy
    except ImportError:
        pass

    import tempfile
    import os
    from collections import defaultdict

    from nltk import compat
    from nltk.data import gzip_open_unicode
    from nltk.util import OrderedDict
    from nltk.probability import DictionaryProbDist

    from nltk.classify.api import ClassifierI
    from nltk.classify.util import CutoffChecker, accuracy, log_likelihood
    from nltk.classify.megam import (call_megam,
                                     write_megam_file, parse_megam_weights)
    from nltk.classify.tadm import call_tadm, write_tadm_file, parse_tadm_weights

    ######################################################################
    #{ Classifier Model
    ######################################################################

    @compat.python_2_unicode_compatible
    class MaxentClassifier(ClassifierI):
        """
        A maximum entropy classifier (also known as a "conditional
        exponential classifier").  This classifier is parameterized by a
        set of "weights", which are used to combine the joint-features
        that are generated from a featureset by an "encoding".  In
        particular, the encoding maps each ``(featureset, label)`` pair to
        a vector.  The probability of each label is then computed using
        the following equation::

                                    dotprod(weights, encode(fs,label))
          prob(fs|label) = ---------------------------------------------------
                           sum(dotprod(weights, encode(fs,l)) for l in labels)

        Where ``dotprod`` is the dot product::

          dotprod(a,b) = sum(x*y for (x,y) in zip(a,b))
        """
        def __init__(self, encoding, weights, logarithmic=True):
            """
            Construct a new maxent classifier model.  Typically, new
            classifier models are created using the ``train()`` method.

            :type encoding: MaxentFeatureEncodingI
            :param encoding: An encoding that is used to convert the
                featuresets that are given to the ``classify`` method into
                joint-feature vectors, which are used by the maxent
                classifier model.

            :type weights: list of float
            :param weights:  The feature weight vector for this classifier.

            :type logarithmic: bool
            :param logarithmic: If false, then use non-logarithmic weights.
            """
            self._encoding = encoding
            self._weights = weights
            self._logarithmic = logarithmic
            #self._logarithmic = False
            assert encoding.length() == len(weights)

        def labels(self):
            return self._encoding.labels()

        def set_weights(self, new_weights):
            """
            Set the feature weight vector for this classifier.
            :param new_weights: The new feature weight vector.
            :type new_weights: list of float
            """
            self._weights = new_weights
            assert self._encoding.length() == len(new_weights)

        def weights(self):
            """
            :return: The feature weight vector for this classifier.
            :rtype: list of float
            """
            return self._weights

        def classify(self, featureset):
            return self.prob_classify(featureset).max()

        def prob_classify(self, featureset):
            prob_dict = {}
            for label in self._encoding.labels():
                feature_vector = self._encoding.encode(featureset, label)

                if self._logarithmic:
                    total = 0.0
                    for (f_id, f_val) in feature_vector:
                        total += self._weights[f_id] * f_val
                    prob_dict[label] = total

                else:
                    prod = 1.0
                    for (f_id, f_val) in feature_vector:
                        prod *= self._weights[f_id] ** f_val
                    prob_dict[label] = prod

            # Normalize the dictionary to give a probability distribution
            return DictionaryProbDist(prob_dict, log=self._logarithmic,
                                      normalize=True)

        def explain(self, featureset, columns=4):
            """
            Print a table showing the effect of each of the features in
            the given feature set, and how they combine to determine the
            probabilities of each label for that featureset.
            """
            descr_width = 50
            TEMPLATE = '  %-'+str(descr_width-2)+'s%s%8.3f'

            pdist = self.prob_classify(featureset)
            labels = sorted(pdist.samples(), key=pdist.prob, reverse=True)
            labels = labels[:columns]
            print('  Feature'.ljust(descr_width)+''.join(
                '%8s' % (("%s" % l)[:7]) for l in labels))
            print('  '+'-'*(descr_width-2+8*len(labels)))
            sums = defaultdict(int)
            for i, label in enumerate(labels):
                feature_vector = self._encoding.encode(featureset, label)
                feature_vector.sort(key=lambda fid__: abs(self._weights[fid__[0]]),
                                    reverse=True)
                for (f_id, f_val) in feature_vector:
                    if self._logarithmic:
                        score = self._weights[f_id] * f_val
                    else: score = self._weights[f_id] ** f_val
                    descr = self._encoding.describe(f_id)
                    descr = descr.split(' and label is ')[0] # hack
                    descr += ' (%s)' % f_val                 # hack
                    if len(descr) > 47:
                        descr = descr[:44]+'...'
                    print(TEMPLATE % (descr, i*8*' ', score))
                    sums[label] += score
            print('  '+'-'*(descr_width-1+8*len(labels)))
            print('  TOTAL:'.ljust(descr_width)+''.join(
                '%8.3f' % sums[l] for l in labels))
            print('  PROBS:'.ljust(descr_width)+''.join(
                '%8.3f' % pdist.prob(l) for l in labels))

        def show_most_informative_features(self, n=10, show='all'):
            """
            :param show: all, neg, or pos (for negative-only or positive-only)
            """
            fids = sorted(list(range(len(self._weights))),
                          key=lambda fid: abs(self._weights[fid]),
                          reverse=True)
            if show == 'pos':
                fids = [fid for fid in fids if self._weights[fid] > 0]
            elif show == 'neg':
                fids = [fid for fid in fids if self._weights[fid] < 0]
            for fid in fids[:n]:
                print('%8.3f %s' % (self._weights[fid],
                                    self._encoding.describe(fid)))

        def __repr__(self):
            return ('<ConditionalExponentialClassifier: %d labels, %d features>' %
                    (len(self._encoding.labels()), self._encoding.length()))

        #: A list of the algorithm names that are accepted for the
        #: ``train()`` method's ``algorithm`` parameter.
        ALGORITHMS = ['GIS', 'IIS', 'MEGAM', 'TADM']

        @classmethod
        def train(cls, train_toks, algorithm=None, trace=3, encoding=None,
                  labels=None, gaussian_prior_sigma=0, **cutoffs):
            """
            Train a new maxent classifier based on the given corpus of
            training samples.  This classifier will have its weights
            chosen to maximize entropy while remaining empirically
            consistent with the training corpus.

            :rtype: MaxentClassifier
            :return: The new maxent classifier

            :type train_toks: list
            :param train_toks: Training data, represented as a list of
                pairs, the first member of which is a featureset,
                and the second of which is a classification label.

            :type algorithm: str
            :param algorithm: A case-insensitive string, specifying which
                algorithm should be used to train the classifier.  The
                following algorithms are currently available.

                - Iterative Scaling Methods: Generalized Iterative Scaling (``'GIS'``),
                  Improved Iterative Scaling (``'IIS'``)
                - External Libraries (requiring megam):
                  LM-BFGS algorithm, with training performed by Megam (``'megam'``)

                The default algorithm is ``'IIS'``.

            :type trace: int
            :param trace: The level of diagnostic tracing output to produce.
                Higher values produce more verbose output.
            :type encoding: MaxentFeatureEncodingI
            :param encoding: A feature encoding, used to convert featuresets
                into feature vectors.  If none is specified, then a
                ``BinaryMaxentFeatureEncoding`` will be built based on the
                features that are attested in the training corpus.
            :type labels: list(str)
            :param labels: The set of possible labels.  If none is given, then
                the set of all labels attested in the training data will be
                used instead.
            :param gaussian_prior_sigma: The sigma value for a gaussian
                prior on model weights.  Currently, this is supported by
                ``megam``. For other algorithms, its value is ignored.
            :param cutoffs: Arguments specifying various conditions under
                which the training should be halted.  (Some of the cutoff
                conditions are not supported by some algorithms.)

                - ``max_iter=v``: Terminate after ``v`` iterations.
                - ``min_ll=v``: Terminate after the negative average
                  log-likelihood drops under ``v``.
                - ``min_lldelta=v``: Terminate if a single iteration improves
                  log likelihood by less than ``v``.
            """
            if algorithm is None:
                algorithm = 'iis'
            for key in cutoffs:
                if key not in ('max_iter', 'min_ll', 'min_lldelta',
                               'max_acc', 'min_accdelta', 'count_cutoff',
                               'norm', 'explicit', 'bernoulli'):
                    raise TypeError('Unexpected keyword arg %r' % key)
            algorithm = algorithm.lower()
            if algorithm == 'iis':
                return train_maxent_classifier_with_iis(
                    train_toks, trace, encoding, labels, **cutoffs)
            elif algorithm == 'gis':
                return train_maxent_classifier_with_gis(
                    train_toks, trace, encoding, labels, **cutoffs)
            elif algorithm == 'megam':
                return train_maxent_classifier_with_megam(
                    train_toks, trace, encoding, labels,
                    gaussian_prior_sigma, **cutoffs)
            elif algorithm == 'tadm':
                kwargs = cutoffs
                kwargs['trace'] = trace
                kwargs['encoding'] = encoding
                kwargs['labels'] = labels
                kwargs['gaussian_prior_sigma'] = gaussian_prior_sigma
                return TadmMaxentClassifier.train(train_toks, **kwargs)
            else:
                raise ValueError('Unknown algorithm %s' % algorithm)

    #: Alias for MaxentClassifier.
    ConditionalExponentialClassifier = MaxentClassifier

    ######################################################################
    #{ Feature Encodings
    ######################################################################

    class MaxentFeatureEncodingI(object):
        """
        A mapping that converts a set of input-feature values to a vector
        of joint-feature values, given a label.  This conversion is
        necessary to translate featuresets into a format that can be used
        by maximum entropy models.

        The set of joint-features used by a given encoding is fixed, and
        each index in the generated joint-feature vectors corresponds to a
        single joint-feature.  The length of the generated joint-feature
        vectors is therefore constant (for a given encoding).

        Because the joint-feature vectors generated by
        ``MaxentFeatureEncodingI`` are typically very sparse, they are
        represented as a list of ``(index, value)`` tuples, specifying the
        value of each non-zero joint-feature.

        Feature encodings are generally created using the ``train()``
        method, which generates an appropriate encoding based on the
        input-feature values and labels that are present in a given
        corpus.
        """
        def encode(self, featureset, label):
            """
            Given a (featureset, label) pair, return the corresponding
            vector of joint-feature values.  This vector is represented as
            a list of ``(index, value)`` tuples, specifying the value of
            each non-zero joint-feature.

            :type featureset: dict
            :rtype: list(tuple(int, int))
            """
            raise NotImplementedError()

        def length(self):
            """
            :return: The size of the fixed-length joint-feature vectors
                that are generated by this encoding.
            :rtype: int
            """
            raise NotImplementedError()

        def labels(self):
            """
            :return: A list of the \"known labels\" -- i.e., all labels
                ``l`` such that ``self.encode(fs,l)`` can be a nonzero
                joint-feature vector for some value of ``fs``.
            :rtype: list
            """
            raise NotImplementedError()

        def describe(self, fid):
            """
            :return: A string describing the value of the joint-feature
                whose index in the generated feature vectors is ``fid``.
            :rtype: str
            """
            raise NotImplementedError()

        def train(cls, train_toks):
            """
            Construct and return new feature encoding, based on a given
            training corpus ``train_toks``.

            :type train_toks: list(tuple(dict, str))
            :param train_toks: Training data, represented as a list of
                pairs, the first member of which is a feature dictionary,
                and the second of which is a classification label.
            """
            raise NotImplementedError()

    class FunctionBackedMaxentFeatureEncoding(MaxentFeatureEncodingI):
        """
        A feature encoding that calls a user-supplied function to map a
        given featureset/label pair to a sparse joint-feature vector.
        """
        def __init__(self, func, length, labels):
            """
            Construct a new feature encoding based on the given function.

            :type func: (callable)
            :param func: A function that takes two arguments, a featureset
                 and a label, and returns the sparse joint feature vector
                 that encodes them::

                     func(featureset, label) -> feature_vector

                 This sparse joint feature vector (``feature_vector``) is a
                 list of ``(index,value)`` tuples.

            :type length: int
            :param length: The size of the fixed-length joint-feature
                vectors that are generated by this encoding.

            :type labels: list
            :param labels: A list of the \"known labels\" for this
                encoding -- i.e., all labels ``l`` such that
                ``self.encode(fs,l)`` can be a nonzero joint-feature vector
                for some value of ``fs``.
            """
            self._length = length
            self._func = func
            self._labels = labels

        def encode(self, featureset, label):
            return self._func(featureset, label)

        def length(self):
            return self._length

        def labels(self):
            return self._labels

        def describe(self, fid):
            return 'no description available'

    class BinaryMaxentFeatureEncoding(MaxentFeatureEncodingI):
        """
        A feature encoding that generates vectors containing a binary
        joint-features of the form:

        |  joint_feat(fs, l) = { 1 if (fs[fname] == fval) and (l == label)
        |                      {
        |                      { 0 otherwise

        Where ``fname`` is the name of an input-feature, ``fval`` is a value
        for that input-feature, and ``label`` is a label.

        Typically, these features are constructed based on a training
        corpus, using the ``train()`` method.  This method will create one
        feature for each combination of ``fname``, ``fval``, and ``label``
        that occurs at least once in the training corpus.

        The ``unseen_features`` parameter can be used to add "unseen-value
        features", which are used whenever an input feature has a value
        that was not encountered in the training corpus.  These features
        have the form:

        |  joint_feat(fs, l) = { 1 if is_unseen(fname, fs[fname])
        |                      {      and l == label
        |                      {
        |                      { 0 otherwise

        Where ``is_unseen(fname, fval)`` is true if the encoding does not
        contain any joint features that are true when ``fs[fname]==fval``.

        The ``alwayson_features`` parameter can be used to add "always-on
        features", which have the form::

        |  joint_feat(fs, l) = { 1 if (l == label)
        |                      {
        |                      { 0 otherwise

        These always-on features allow the maxent model to directly model
        the prior probabilities of each label.
        """
        def __init__(self, labels, mapping, unseen_features=False,
                     alwayson_features=False):
            """
            :param labels: A list of the \"known labels\" for this encoding.

            :param mapping: A dictionary mapping from ``(fname,fval,label)``
                tuples to corresponding joint-feature indexes.  These
                indexes must be the set of integers from 0...len(mapping).
                If ``mapping[fname,fval,label]=id``, then
                ``self.encode(..., fname:fval, ..., label)[id]`` is 1;
                otherwise, it is 0.

            :param unseen_features: If true, then include unseen value
               features in the generated joint-feature vectors.

            :param alwayson_features: If true, then include always-on
               features in the generated joint-feature vectors.
            """
            if set(mapping.values()) != set(range(len(mapping))):
                raise ValueError('Mapping values must be exactly the '
                                 'set of integers from 0...len(mapping)')

            self._labels = list(labels)
            """A list of attested labels."""

            self._mapping = mapping
            """dict mapping from (fname,fval,label) -> fid"""

            self._length = len(mapping)
            """The length of generated joint feature vectors."""

            self._alwayson = None
            """dict mapping from label -> fid"""

            self._unseen = None
            """dict mapping from fname -> fid"""

            if alwayson_features:
                self._alwayson = dict((label, i+self._length)
                                      for (i, label) in enumerate(labels))
                self._length += len(self._alwayson)

            if unseen_features:
                fnames = set(fname for (fname, fval, label) in mapping)
                self._unseen = dict((fname, i+self._length)
                                    for (i, fname) in enumerate(fnames))
                self._length += len(fnames)

        def encode(self, featureset, label):
            # Inherit docs.
            encoding = []

            # Convert input-features to joint-features:
            for fname, fval in featureset.items():
                # Known feature name & value:
                if (fname, fval, label) in self._mapping:
                    encoding.append((self._mapping[fname, fval, label], 1))

                # Otherwise, we might want to fire an "unseen-value feature".
                elif self._unseen:
                    # Have we seen this fname/fval combination with any label?
                    for label2 in self._labels:
                        if (fname, fval, label2) in self._mapping:
                            break # we've seen this fname/fval combo
                    # We haven't -- fire the unseen-value feature
                    else:
                        if fname in self._unseen:
                            encoding.append((self._unseen[fname], 1))

            # Add always-on features:
            if self._alwayson and label in self._alwayson:
                encoding.append((self._alwayson[label], 1))

            return encoding

        def describe(self, f_id):
            # Inherit docs.
            if not isinstance(f_id, compat.integer_types):
                raise TypeError('describe() expected an int')
            try:
                self._inv_mapping
            except AttributeError:
                self._inv_mapping = [-1]*len(self._mapping)
                for (info, i) in self._mapping.items():
                    self._inv_mapping[i] = info

            if f_id < len(self._mapping):
                (fname, fval, label) = self._inv_mapping[f_id]
                return '%s==%r and label is %r' % (fname, fval, label)
            elif self._alwayson and f_id in self._alwayson.values():
                for (label, f_id2) in self._alwayson.items():
                    if f_id == f_id2:
                        return 'label is %r' % label
            elif self._unseen and f_id in self._unseen.values():
                for (fname, f_id2) in self._unseen.items():
                    if f_id == f_id2:
                        return '%s is unseen' % fname
            else:
                raise ValueError('Bad feature id')

        def labels(self):
            # Inherit docs.
            return self._labels

        def length(self):
            # Inherit docs.
            return self._length

        @classmethod
        def train(cls, train_toks, count_cutoff=0, labels=None, **options):
            """
            Construct and return new feature encoding, based on a given
            training corpus ``train_toks``.  See the class description
            ``BinaryMaxentFeatureEncoding`` for a description of the
            joint-features that will be included in this encoding.

            :type train_toks: list(tuple(dict, str))
            :param train_toks: Training data, represented as a list of
                pairs, the first member of which is a feature dictionary,
                and the second of which is a classification label.

            :type count_cutoff: int
            :param count_cutoff: A cutoff value that is used to discard
                rare joint-features.  If a joint-feature's value is 1
                fewer than ``count_cutoff`` times in the training corpus,
                then that joint-feature is not included in the generated
                encoding.

            :type labels: list
            :param labels: A list of labels that should be used by the
                classifier.  If not specified, then the set of labels
                attested in ``train_toks`` will be used.

            :param options: Extra parameters for the constructor, such as
                ``unseen_features`` and ``alwayson_features``.
            """
            mapping = {}              # maps (fname, fval, label) -> fid
            seen_labels = set()       # The set of labels we've encountered
            count = defaultdict(int)  # maps (fname, fval) -> count

            for (tok, label) in train_toks:
                if labels and label not in labels:
                    raise ValueError('Unexpected label %s' % label)
                seen_labels.add(label)

                # Record each of the features.
                for (fname, fval) in tok.items():

                    # If a count cutoff is given, then only add a joint
                    # feature once the corresponding (fname, fval, label)
                    # tuple exceeds that cutoff.
                    count[fname, fval] += 1
                    if count[fname, fval] >= count_cutoff:
                        if (fname, fval, label) not in mapping:
                            mapping[fname, fval, label] = len(mapping)

            if labels is None:
                labels = seen_labels
            return cls(labels, mapping, **options)

    class GISEncoding(BinaryMaxentFeatureEncoding):
        """
        A binary feature encoding which adds one new joint-feature to the
        joint-features defined by ``BinaryMaxentFeatureEncoding``: a
        correction feature, whose value is chosen to ensure that the
        sparse vector always sums to a constant non-negative number.  This
        new feature is used to ensure two preconditions for the GIS
        training algorithm:

          - At least one feature vector index must be nonzero for every
            token.
          - The feature vector must sum to a constant non-negative number
            for every token.
        """
        def __init__(self, labels, mapping, unseen_features=False,
                     alwayson_features=False, C=None):
            """
            :param C: The correction constant.  The value of the correction
                feature is based on this value.  In particular, its value is
                ``C - sum([v for (f,v) in encoding])``.
            :seealso: ``BinaryMaxentFeatureEncoding.__init__``
            """
            BinaryMaxentFeatureEncoding.__init__(
                self, labels, mapping, unseen_features, alwayson_features)
            if C is None:
                C = len(set(fname for (fname, fval, label) in mapping))+1
            self._C = C

        @property
        def C(self):
            """The non-negative constant that all encoded feature vectors
            will sum to."""
            return self._C

        def encode(self, featureset, label):
            # Get the basic encoding.
            encoding = BinaryMaxentFeatureEncoding.encode(self, featureset, label)
            base_length = BinaryMaxentFeatureEncoding.length(self)

            # Add a correction feature.
            total = sum(v for (f, v) in encoding)
            if total >= self._C:
                raise ValueError('Correction feature is not high enough!')
            encoding.append((base_length, self._C-total))

            # Return the result
            return encoding

        def length(self):
            return BinaryMaxentFeatureEncoding.length(self) + 1

        def describe(self, f_id):
            if f_id == BinaryMaxentFeatureEncoding.length(self):
                return 'Correction feature (%s)' % self._C
            else:
                return BinaryMaxentFeatureEncoding.describe(self, f_id)

    class TadmEventMaxentFeatureEncoding(BinaryMaxentFeatureEncoding):
        def __init__(self, labels, mapping, unseen_features=False,
                           alwayson_features=False):
            self._mapping = OrderedDict(mapping)
            self._label_mapping = OrderedDict()
            BinaryMaxentFeatureEncoding.__init__(self, labels, self._mapping,
                                                 unseen_features,
                                                 alwayson_features)

        def encode(self, featureset, label):
            encoding = []
            for feature, value in featureset.items():
                if (feature, label) not in self._mapping:
                    self._mapping[(feature, label)] = len(self._mapping)
                if value not in self._label_mapping:
                    if not isinstance(value, int):
                        self._label_mapping[value] = len(self._label_mapping)
                    else:
                        self._label_mapping[value] = value
                encoding.append((self._mapping[(feature, label)],
                                 self._label_mapping[value]))
            return encoding

        def labels(self):
            return self._labels

        def describe(self, fid):
            for (feature, label) in self._mapping:
                if self._mapping[(feature, label)] == fid:
                    return (feature, label)

        def length(self):
            return len(self._mapping)

        @classmethod
        def train(cls, train_toks, count_cutoff=0, labels=None, **options):
            mapping = OrderedDict()
            if not labels:
                labels = []

            # This gets read twice, so compute the values in case it's lazy.
            train_toks = list(train_toks)

            for (featureset, label) in train_toks:
                if label not in labels:
                    labels.append(label)

            for (featureset, label) in train_toks:
                for label in labels:
                    for feature in featureset:
                        if (feature, label) not in mapping:
                            mapping[(feature, label)] = len(mapping)

            return cls(labels, mapping, **options)

    class TypedMaxentFeatureEncoding(MaxentFeatureEncodingI):
        """
        A feature encoding that generates vectors containing integer,
        float and binary joint-features of the form:

        Binary (for string and boolean features):

        |  joint_feat(fs, l) = { 1 if (fs[fname] == fval) and (l == label)
        |                      {
        |                      { 0 otherwise

        Value (for integer and float features):

        |  joint_feat(fs, l) = { fval if     (fs[fname] == type(fval))
        |                      {         and (l == label)
        |                      {
        |                      { not encoded otherwise

        Where ``fname`` is the name of an input-feature, ``fval`` is a value
        for that input-feature, and ``label`` is a label.

        Typically, these features are constructed based on a training
        corpus, using the ``train()`` method.

        For string and boolean features [type(fval) not in (int, float)]
        this method will create one feature for each combination of
        ``fname``, ``fval``, and ``label`` that occurs at least once in the
        training corpus.

        For integer and float features [type(fval) in (int, float)] this
        method will create one feature for each combination of ``fname``
        and ``label`` that occurs at least once in the training corpus.

        For binary features the ``unseen_features`` parameter can be used
        to add "unseen-value features", which are used whenever an input
        feature has a value that was not encountered in the training
        corpus.  These features have the form:

        |  joint_feat(fs, l) = { 1 if is_unseen(fname, fs[fname])
        |                      {      and l == label
        |                      {
        |                      { 0 otherwise

        Where ``is_unseen(fname, fval)`` is true if the encoding does not
        contain any joint features that are true when ``fs[fname]==fval``.

        The ``alwayson_features`` parameter can be used to add "always-on
        features", which have the form:

        |  joint_feat(fs, l) = { 1 if (l == label)
        |                      {
        |                      { 0 otherwise

        These always-on features allow the maxent model to directly model
        the prior probabilities of each label.
        """
        def __init__(self, labels, mapping, unseen_features=False,
                     alwayson_features=False):
            """
            :param labels: A list of the \"known labels\" for this encoding.

            :param mapping: A dictionary mapping from ``(fname,fval,label)``
                tuples to corresponding joint-feature indexes.  These
                indexes must be the set of integers from 0...len(mapping).
                If ``mapping[fname,fval,label]=id``, then
                ``self.encode({..., fname:fval, ...``, label)[id]} is 1;
                otherwise, it is 0.

            :param unseen_features: If true, then include unseen value
               features in the generated joint-feature vectors.

            :param alwayson_features: If true, then include always-on
               features in the generated joint-feature vectors.
            """
            if set(mapping.values()) != set(range(len(mapping))):
                raise ValueError('Mapping values must be exactly the '
                                 'set of integers from 0...len(mapping)')

            self._labels = list(labels)
            """A list of attested labels."""

            self._mapping = mapping
            """dict mapping from (fname,fval,label) -> fid"""

            self._length = len(mapping)
            """The length of generated joint feature vectors."""

            self._alwayson = None
            """dict mapping from label -> fid"""

            self._unseen = None
            """dict mapping from fname -> fid"""

            if alwayson_features:
                self._alwayson = dict((label, i+self._length)
                                      for (i, label) in enumerate(labels))
                self._length += len(self._alwayson)

            if unseen_features:
                fnames = set(fname for (fname, fval, label) in mapping)
                self._unseen = dict((fname, i+self._length)
                                    for (i, fname) in enumerate(fnames))
                self._length += len(fnames)

        def encode(self, featureset, label):
            # Inherit docs.
            encoding = []

            # Convert input-features to joint-features:
            for fname, fval in featureset.items():
                if isinstance(fval, (compat.integer_types, float)):
                    # Known feature name & value:
                    if (fname, type(fval), label) in self._mapping:
                        encoding.append((self._mapping[fname, type(fval),
                                                       label], fval))
                else:
                    # Known feature name & value:
                    if (fname, fval, label) in self._mapping:
                        encoding.append((self._mapping[fname, fval, label], 1))

                    # Otherwise, we might want to fire an "unseen-value feature".
                    elif self._unseen:
                        # Have we seen this fname/fval combination with any label?
                        for label2 in self._labels:
                            if (fname, fval, label2) in self._mapping:
                                break # we've seen this fname/fval combo
                        # We haven't -- fire the unseen-value feature
                        else:
                            if fname in self._unseen:
                                encoding.append((self._unseen[fname], 1))

            # Add always-on features:
            if self._alwayson and label in self._alwayson:
                encoding.append((self._alwayson[label], 1))

            return encoding

        def describe(self, f_id):
            # Inherit docs.
            if not isinstance(f_id, compat.integer_types):
                raise TypeError('describe() expected an int')
            try:
                self._inv_mapping
            except AttributeError:
                self._inv_mapping = [-1]*len(self._mapping)
                for (info, i) in self._mapping.items():
                    self._inv_mapping[i] = info

            if f_id < len(self._mapping):
                (fname, fval, label) = self._inv_mapping[f_id]
                return '%s==%r and label is %r' % (fname, fval, label)
            elif self._alwayson and f_id in self._alwayson.values():
                for (label, f_id2) in self._alwayson.items():
                    if f_id == f_id2:
                        return 'label is %r' % label
            elif self._unseen and f_id in self._unseen.values():
                for (fname, f_id2) in self._unseen.items():
                    if f_id == f_id2:
                        return '%s is unseen' % fname
            else:
                raise ValueError('Bad feature id')

        def labels(self):
            # Inherit docs.
            return self._labels

        def length(self):
            # Inherit docs.
            return self._length

        @classmethod
        def train(cls, train_toks, count_cutoff=0, labels=None, **options):
            """
            Construct and return new feature encoding, based on a given
            training corpus ``train_toks``.  See the class description
            ``TypedMaxentFeatureEncoding`` for a description of the
            joint-features that will be included in this encoding.

            Note: recognized feature values types are (int, float), over
            types are interpreted as regular binary features.

            :type train_toks: list(tuple(dict, str))
            :param train_toks: Training data, represented as a list of
                pairs, the first member of which is a feature dictionary,
                and the second of which is a classification label.

            :type count_cutoff: int
            :param count_cutoff: A cutoff value that is used to discard
                rare joint-features.  If a joint-feature's value is 1
                fewer than ``count_cutoff`` times in the training corpus,
                then that joint-feature is not included in the generated
                encoding.

            :type labels: list
            :param labels: A list of labels that should be used by the
                classifier.  If not specified, then the set of labels
                attested in ``train_toks`` will be used.

            :param options: Extra parameters for the constructor, such as
                ``unseen_features`` and ``alwayson_features``.
            """
            mapping = {}              # maps (fname, fval, label) -> fid
            seen_labels = set()       # The set of labels we've encountered
            count = defaultdict(int)  # maps (fname, fval) -> count

            for (tok, label) in train_toks:
                if labels and label not in labels:
                    raise ValueError('Unexpected label %s' % label)
                seen_labels.add(label)

                # Record each of the features.
                for (fname, fval) in tok.items():
                    if type(fval) in (int, float):
                        fval = type(fval)
                    # If a count cutoff is given, then only add a joint
                    # feature once the corresponding (fname, fval, label)
                    # tuple exceeds that cutoff.
                    count[fname, fval] += 1
                    if count[fname, fval] >= count_cutoff:
                        if (fname, fval, label) not in mapping:
                            mapping[fname, fval, label] = len(mapping)

            if labels is None:
                labels = seen_labels
            return cls(labels, mapping, **options)

    ######################################################################
    #{ Classifier Trainer: Generalized Iterative Scaling
    ######################################################################

    def train_maxent_classifier_with_gis(train_toks, trace=3, encoding=None,
                                         labels=None, **cutoffs):
        """
        Train a new ``ConditionalExponentialClassifier``, using the given
        training samples, using the Generalized Iterative Scaling
        algorithm.  This ``ConditionalExponentialClassifier`` will encode
        the model that maximizes entropy from all the models that are
        empirically consistent with ``train_toks``.

        :see: ``train_maxent_classifier()`` for parameter descriptions.
        """
        cutoffs.setdefault('max_iter', 100)
        cutoffchecker = CutoffChecker(cutoffs)

        # Construct an encoding from the training data.
        if encoding is None:
            encoding = GISEncoding.train(train_toks, labels=labels)

        if not hasattr(encoding, 'C'):
            raise TypeError('The GIS algorithm requires an encoding that '
                            'defines C (e.g., GISEncoding).')

        # Cinv is the inverse of the sum of each joint feature vector.
        # This controls the learning rate: higher Cinv (or lower C) gives
        # faster learning.
        Cinv = 1.0/encoding.C

        # Count how many times each feature occurs in the training data.
        empirical_fcount = calculate_empirical_fcount(train_toks, encoding)

        # Check for any features that are not attested in train_toks.
        unattested = set(numpy.nonzero(empirical_fcount == 0)[0])

        # Build the classifier.  Start with weight=0 for each attested
        # feature, and weight=-infinity for each unattested feature.
        weights = numpy.zeros(len(empirical_fcount), 'd')
        for fid in unattested:
            weights[fid] = numpy.NINF
        classifier = ConditionalExponentialClassifier(encoding, weights)

        # Take the log of the empirical fcount.
        log_empirical_fcount = numpy.log2(empirical_fcount)
        del empirical_fcount

        # Old log-likelihood and accuracy; used to check if the change
        # in log-likelihood or accuracy is sufficient to indicate convergence.
        ll_old = None
        acc_old = None

        if trace > 0:
            print('  ==> Training (%d iterations)' % cutoffs['max_iter'])
        if trace > 2:
            print()
            print('      Iteration    Log Likelihood    Accuracy')
            print('      ---------------------------------------')

        # Train the classifier.
        try:
            while True:
                if trace > 2:
                    ll = cutoffchecker.ll or log_likelihood(classifier, train_toks)
                    acc = cutoffchecker.acc or accuracy(classifier, train_toks)
                    iternum = cutoffchecker.iter
                    print('     %9d    %14.5f    %9.3f' % (iternum, ll, acc))

                # Use the model to estimate the number of times each
                # feature should occur in the training data.
                estimated_fcount = calculate_estimated_fcount(
                    classifier, train_toks, encoding)

                # Take the log of estimated fcount (avoid taking log(0).)
                for fid in unattested:
                    estimated_fcount[fid] += 1
                log_estimated_fcount = numpy.log2(estimated_fcount)
                del estimated_fcount

                # Update the classifier weights
                weights = classifier.weights()
                weights += (log_empirical_fcount - log_estimated_fcount) * Cinv
                classifier.set_weights(weights)

                # Check the log-likelihood & accuracy cutoffs.
                if cutoffchecker.check(classifier, train_toks):
                    break

        except KeyboardInterrupt:
            print('      Training stopped: keyboard interrupt')
        except:
            raise

        if trace > 2:
            ll = log_likelihood(classifier, train_toks)
            acc = accuracy(classifier, train_toks)
            print('         Final    %14.5f    %9.3f' % (ll, acc))

    # Return the classifier.
        return classifier

    def calculate_empirical_fcount(train_toks, encoding):
        fcount = numpy.zeros(encoding.length(), 'd')

        for tok, label in train_toks:
            for (index, val) in encoding.encode(tok, label):
                fcount[index] += val

        return fcount

    def calculate_estimated_fcount(classifier, train_toks, encoding):
        fcount = numpy.zeros(encoding.length(), 'd')

        for tok, label in train_toks:
            pdist = classifier.prob_classify(tok)
            for label in pdist.samples():
                prob = pdist.prob(label)
                for (fid, fval) in encoding.encode(tok, label):
                    fcount[fid] += prob*fval

        return fcount

    ######################################################################
    #{ Classifier Trainer: Improved Iterative Scaling
    ######################################################################

    def train_maxent_classifier_with_iis(train_toks, trace=3, encoding=None,
                                         labels=None, **cutoffs):
        """
        Train a new ``ConditionalExponentialClassifier``, using the given
        training samples, using the Improved Iterative Scaling algorithm.
        This ``ConditionalExponentialClassifier`` will encode the model
        that maximizes entropy from all the models that are empirically
        consistent with ``train_toks``.

        :see: ``train_maxent_classifier()`` for parameter descriptions.
        """
        cutoffs.setdefault('max_iter', 100)
        cutoffchecker = CutoffChecker(cutoffs)

        # Construct an encoding from the training data.
        if encoding is None:
            encoding = BinaryMaxentFeatureEncoding.train(train_toks, labels=labels)

        # Count how many times each feature occurs in the training data.
        empirical_ffreq = (calculate_empirical_fcount(train_toks, encoding) /
                           len(train_toks))

        # Find the nf map, and related variables nfarray and nfident.
        # nf is the sum of the features for a given labeled text.
        # nfmap compresses this sparse set of values to a dense list.
        # nfarray performs the reverse operation.  nfident is
        # nfarray multiplied by an identity matrix.
        nfmap = calculate_nfmap(train_toks, encoding)
        nfarray = numpy.array(sorted(nfmap, key=nfmap.__getitem__), 'd')
        nftranspose = numpy.reshape(nfarray, (len(nfarray), 1))

        # Check for any features that are not attested in train_toks.
        unattested = set(numpy.nonzero(empirical_ffreq == 0)[0])

        # Build the classifier.  Start with weight=0 for each attested
        # feature, and weight=-infinity for each unattested feature.
        weights = numpy.zeros(len(empirical_ffreq), 'd')
        for fid in unattested:
            weights[fid] = numpy.NINF
        classifier = ConditionalExponentialClassifier(encoding, weights)

        if trace > 0:
            print('  ==> Training (%d iterations)' % cutoffs['max_iter'])
        if trace > 2:
            print()
            print('      Iteration    Log Likelihood    Accuracy')
            print('      ---------------------------------------')

        # Old log-likelihood and accuracy; used to check if the change
        # in log-likelihood or accuracy is sufficient to indicate convergence.
        ll_old = None
        acc_old = None

        # Train the classifier.
        try:
            while True:
                if trace > 2:
                    ll = cutoffchecker.ll or log_likelihood(classifier, train_toks)
                    acc = cutoffchecker.acc or accuracy(classifier, train_toks)
                    iternum = cutoffchecker.iter
                    print('     %9d    %14.5f    %9.3f' % (iternum, ll, acc))

                # Calculate the deltas for this iteration, using Newton's method.
                deltas = calculate_deltas(
                    train_toks, classifier, unattested, empirical_ffreq,
                    nfmap, nfarray, nftranspose, encoding)

                # Use the deltas to update our weights.
                weights = classifier.weights()
                weights += deltas
                classifier.set_weights(weights)

                # Check the log-likelihood & accuracy cutoffs.
                if cutoffchecker.check(classifier, train_toks):
                    break

        except KeyboardInterrupt:
            print('      Training stopped: keyboard interrupt')
        except:
            raise

        if trace > 2:
            ll = log_likelihood(classifier, train_toks)
            acc = accuracy(classifier, train_toks)
            print('         Final    %14.5f    %9.3f' % (ll, acc))

        # Return the classifier.
        return classifier

    def calculate_nfmap(train_toks, encoding):
        """
        Construct a map that can be used to compress ``nf`` (which is
        typically sparse).

        *nf(feature_vector)* is the sum of the feature values for
        *feature_vector*.

        This represents the number of features that are active for a
        given labeled text.  This method finds all values of *nf(t)*
        that are attested for at least one token in the given list of
        training tokens; and constructs a dictionary mapping these
        attested values to a continuous range *0...N*.  For example,
        if the only values of *nf()* that were attested were 3, 5, and
        7, then ``_nfmap`` might return the dictionary ``{3:0, 5:1, 7:2}``.

        :return: A map that can be used to compress ``nf`` to a dense
            vector.
        :rtype: dict(int -> int)
        """
        # Map from nf to indices.  This allows us to use smaller arrays.
        nfset = set()
        for tok, _ in train_toks:
            for label in encoding.labels():
                nfset.add(sum(val for (id, val) in encoding.encode(tok, label)))
        return dict((nf, i) for (i, nf) in enumerate(nfset))

    def calculate_deltas(train_toks, classifier, unattested, ffreq_empirical,
                         nfmap, nfarray, nftranspose, encoding):
        """
        Calculate the update values for the classifier weights for
        this iteration of IIS.  These update weights are the value of
        ``delta`` that solves the equation::

          ffreq_empirical[i]
                 =
          SUM[fs,l] (classifier.prob_classify(fs).prob(l) *
                     feature_vector(fs,l)[i] *
                     exp(delta[i] * nf(feature_vector(fs,l))))

        Where:
            - *(fs,l)* is a (featureset, label) tuple from ``train_toks``
            - *feature_vector(fs,l)* = ``encoding.encode(fs,l)``
            - *nf(vector)* = ``sum([val for (id,val) in vector])``

        This method uses Newton's method to solve this equation for
        *delta[i]*.  In particular, it starts with a guess of
        ``delta[i]`` = 1; and iteratively updates ``delta`` with:

        | delta[i] -= (ffreq_empirical[i] - sum1[i])/(-sum2[i])

        until convergence, where *sum1* and *sum2* are defined as:

        |    sum1[i](delta) = SUM[fs,l] f[i](fs,l,delta)
        |    sum2[i](delta) = SUM[fs,l] (f[i](fs,l,delta).nf(feature_vector(fs,l)))
        |    f[i](fs,l,delta) = (classifier.prob_classify(fs).prob(l) .
        |                        feature_vector(fs,l)[i] .
        |                        exp(delta[i] . nf(feature_vector(fs,l))))

        Note that *sum1* and *sum2* depend on ``delta``; so they need
        to be re-computed each iteration.

        The variables ``nfmap``, ``nfarray``, and ``nftranspose`` are
        used to generate a dense encoding for *nf(ltext)*.  This
        allows ``_deltas`` to calculate *sum1* and *sum2* using
        matrices, which yields a significant performance improvement.

        :param train_toks: The set of training tokens.
        :type train_toks: list(tuple(dict, str))
        :param classifier: The current classifier.
        :type classifier: ClassifierI
        :param ffreq_empirical: An array containing the empirical
            frequency for each feature.  The *i*\ th element of this
            array is the empirical frequency for feature *i*.
        :type ffreq_empirical: sequence of float
        :param unattested: An array that is 1 for features that are
            not attested in the training data; and 0 for features that
            are attested.  In other words, ``unattested[i]==0`` iff
            ``ffreq_empirical[i]==0``.
        :type unattested: sequence of int
        :param nfmap: A map that can be used to compress ``nf`` to a dense
            vector.
        :type nfmap: dict(int -> int)
        :param nfarray: An array that can be used to uncompress ``nf``
            from a dense vector.
        :type nfarray: array(float)
        :param nftranspose: The transpose of ``nfarray``
        :type nftranspose: array(float)
        """
        # These parameters control when we decide that we've
        # converged.  It probably should be possible to set these
        # manually, via keyword arguments to train.
        NEWTON_CONVERGE = 1e-12
        MAX_NEWTON = 300

        deltas = numpy.ones(encoding.length(), 'd')

        # Precompute the A matrix:
        # A[nf][id] = sum ( p(fs) * p(label|fs) * f(fs,label) )
        # over all label,fs s.t. num_features[label,fs]=nf
        A = numpy.zeros((len(nfmap), encoding.length()), 'd')

        for tok, label in train_toks:
            dist = classifier.prob_classify(tok)

            for label in encoding.labels():
                # Generate the feature vector
                feature_vector = encoding.encode(tok, label)
                # Find the number of active features
                nf = sum(val for (id, val) in feature_vector)
                # Update the A matrix
                for (id, val) in feature_vector:
                    A[nfmap[nf], id] += dist.prob(label) * val
        A /= len(train_toks)

        # Iteratively solve for delta.  Use the following variables:
        #   - nf_delta[x][y] = nfarray[x] * delta[y]
        #   - exp_nf_delta[x][y] = exp(nf[x] * delta[y])
        #   - nf_exp_nf_delta[x][y] = nf[x] * exp(nf[x] * delta[y])
        #   - sum1[i][nf] = sum p(fs)p(label|fs)f[i](label,fs)
        #                       exp(delta[i]nf)
        #   - sum2[i][nf] = sum p(fs)p(label|fs)f[i](label,fs)
        #                       nf exp(delta[i]nf)
        for rangenum in range(MAX_NEWTON):
            nf_delta = numpy.outer(nfarray, deltas)
            exp_nf_delta = 2 ** nf_delta
            nf_exp_nf_delta = nftranspose * exp_nf_delta
            sum1 = numpy.sum(exp_nf_delta * A, axis=0)
            sum2 = numpy.sum(nf_exp_nf_delta * A, axis=0)

            # Avoid division by zero.
            for fid in unattested:
                sum2[fid] += 1

            # Update the deltas.
            deltas -= (ffreq_empirical - sum1) / -sum2

            # We can stop once we converge.
            n_error = (numpy.sum(abs((ffreq_empirical-sum1)))/
                       numpy.sum(abs(deltas)))
            if n_error < NEWTON_CONVERGE:
                return deltas

        return deltas

    ######################################################################
    #{ Classifier Trainer: megam
    ######################################################################

    # [xx] possible extension: add support for using implicit file format;
    # this would need to put requirements on what encoding is used.  But
    # we may need this for other maxent classifier trainers that require
    # implicit formats anyway.
    def train_maxent_classifier_with_megam(train_toks, trace=3, encoding=None,
                                           labels=None, gaussian_prior_sigma=0,
                                           **kwargs):
        """
        Train a new ``ConditionalExponentialClassifier``, using the given
        training samples, using the external ``megam`` library.  This
        ``ConditionalExponentialClassifier`` will encode the model that
        maximizes entropy from all the models that are empirically
        consistent with ``train_toks``.

        :see: ``train_maxent_classifier()`` for parameter descriptions.
        :see: ``nltk.classify.megam``
        """

        explicit = True
        bernoulli = True
        if 'explicit' in kwargs:
            explicit = kwargs['explicit']
        if 'bernoulli' in kwargs:
            bernoulli = kwargs['bernoulli']

        # Construct an encoding from the training data.
        if encoding is None:
            # Count cutoff can also be controlled by megam with the -minfc
            # option. Not sure where the best place for it is.
            count_cutoff = kwargs.get('count_cutoff', 0)
            encoding = BinaryMaxentFeatureEncoding.train(train_toks, count_cutoff,
                                                         labels=labels,
                                                         alwayson_features=True)
        elif labels is not None:
            raise ValueError('Specify encoding or labels, not both')

        # Write a training file for megam.
        try:
            fd, trainfile_name = tempfile.mkstemp(prefix='nltk-')
            with open(trainfile_name, 'w') as trainfile:
                write_megam_file(train_toks, encoding, trainfile,
                                 explicit=explicit, bernoulli=bernoulli)
            os.close(fd)
        except (OSError, IOError, ValueError) as e:
            raise ValueError('Error while creating megam training file: %s' % e)

        # Run megam on the training file.
        options = []
        options += ['-nobias', '-repeat', '10']
        if explicit:
            options += ['-explicit']
        if not bernoulli:
            options += ['-fvals']
        if gaussian_prior_sigma:
            # Lambda is just the precision of the Gaussian prior, i.e. it's the
            # inverse variance, so the parameter conversion is 1.0/sigma**2.
            # See http://www.umiacs.umd.edu/~hal/docs/daume04cg-bfgs.pdf.
            inv_variance = 1.0 / gaussian_prior_sigma**2
        else:
            inv_variance = 0
        options += ['-lambda', '%.2f' % inv_variance, '-tune']
        if trace < 3:
            options += ['-quiet']
        if 'max_iter' in kwargs:
            options += ['-maxi', '%s' % kwargs['max_iter']]
        if 'll_delta' in kwargs:
            # [xx] this is actually a perplexity delta, not a log
            # likelihood delta
            options += ['-dpp', '%s' % abs(kwargs['ll_delta'])]
        if hasattr(encoding, 'cost'):
            options += ['-multilabel']  # each possible la
        options += ['multiclass', trainfile_name]
        stdout = call_megam(options)
        # print './megam_i686.opt ', ' '.join(options)
        # Delete the training file
        try:
            os.remove(trainfile_name)
        except (OSError, IOError) as e:
            print('Warning: unable to delete %s: %s' % (trainfile_name, e))

        # Parse the generated weight vector.
        weights = parse_megam_weights(stdout, encoding.length(), explicit)

        # Convert from base-e to base-2 weights.
        weights *= numpy.log2(numpy.e)

        # Build the classifier
        return MaxentClassifier(encoding, weights)

    ######################################################################
    #{ Classifier Trainer: tadm
    ######################################################################

    class TadmMaxentClassifier(MaxentClassifier):
        @classmethod
        def train(cls, train_toks, **kwargs):
            algorithm = kwargs.get('algorithm', 'tao_lmvm')
            trace = kwargs.get('trace', 3)
            encoding = kwargs.get('encoding', None)
            labels = kwargs.get('labels', None)
            sigma = kwargs.get('gaussian_prior_sigma', 0)
            count_cutoff = kwargs.get('count_cutoff', 0)
            max_iter = kwargs.get('max_iter')
            ll_delta = kwargs.get('min_lldelta')

            # Construct an encoding from the training data.
            if not encoding:
                encoding = TadmEventMaxentFeatureEncoding.train(train_toks,
                                                                count_cutoff,
                                                                labels=labels)

            trainfile_fd, trainfile_name = \
                tempfile.mkstemp(prefix='nltk-tadm-events-', suffix='.gz')
            weightfile_fd, weightfile_name = \
                tempfile.mkstemp(prefix='nltk-tadm-weights-')

            trainfile = gzip_open_unicode(trainfile_name, 'w')
            write_tadm_file(train_toks, encoding, trainfile)
            trainfile.close()

            options = []
            options.extend(['-monitor'])
            options.extend(['-method', algorithm])
            if sigma:
                options.extend(['-l2', '%.6f' % sigma**2])
            if max_iter:
                options.extend(['-max_it', '%d' % max_iter])
            if ll_delta:
                options.extend(['-fatol', '%.6f' % abs(ll_delta)])
            options.extend(['-events_in', trainfile_name])
            options.extend(['-params_out', weightfile_name])
            if trace < 3:
                options.extend(['2>&1'])
            else:
                options.extend(['-summary'])

            call_tadm(options)

            with open(weightfile_name, 'r') as weightfile:
                weights = parse_tadm_weights(weightfile)

            os.remove(trainfile_name)
            os.remove(weightfile_name)

            # Convert from base-e to base-2 weights.
            weights *= numpy.log2(numpy.e)

            # Build the classifier
            return cls(encoding, weights)

    ######################################################################
    #{ Demo
    ######################################################################
    def demo():
        from nltk.classify.util import names_demo
        classifier = names_demo(MaxentClassifier.train)

    if __name__ == '__main__':
        demo()