imodels.rule_set.boosted_rules API documentation

# Module `imodels.rule_set.boosted_rules`

Expand source code

    from copy import deepcopy
    from functools import partial

    import numpy as np
    from sklearn.base import BaseEstimator, ClassifierMixin, MetaEstimatorMixin
    from sklearn.preprocessing import normalize
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.utils.multiclass import check_classification_targets, unique_labels
    from sklearn.utils.validation import check_X_y, check_array, check_is_fitted

    from imodels.rule_set.rule_set import RuleSet
    from imodels.rule_set.slipper_util import SlipperBaseEstimator
    from imodels.util.convert import tree_to_code, tree_to_rules, dict_to_rule
    from imodels.util.rule import Rule, get_feature_dict, replace_feature_name

    class BoostedRulesClassifier(RuleSet, BaseEstimator, MetaEstimatorMixin, ClassifierMixin):
        '''An easy-interpretable classifier optimizing simple logical rules.
        Currently limited to only binary classification.

        Params
        ------
        estimator: object with fit and predict methods
            Defaults to DecisionTreeClassifier with AdaBoost.
            For SLIPPER, should pass estimator=imodels.SlipperBaseEstimator
        '''

        def __init__(self, n_estimators=10, estimator=partial(DecisionTreeClassifier, max_depth=1), random_state=0):
            self.n_estimators = n_estimators
            self.estimator = estimator
            self.random_state = random_state

        def fit(self, X, y, feature_names=None, sample_weight=None):
            """Fit the model according to the given training data.

            Parameters
            ----------
            X : array-like, shape (n_samples, n_features)
                Training vector, where n_samples is the number of samples and
                n_features is the number of features.

            y : array-like, shape (n_samples,)
                Target vector relative to X. Has to follow the convention 0 for
                normal data, 1 for anomalies.

            sample_weight : array-like, shape (n_samples,) optional
                Array of weights that are assigned to individual samples, typically
                the amount in case of transactions data. Used to grow regression
                trees producing further rules to be tested.
                If not provided, then each sample is given unit weight.

            Returns
            -------
            self : object
                Returns self.
            """

            X, y = check_X_y(X, y)
            check_classification_targets(y)
            self.n_features_ = X.shape[1]
            self.classes_ = unique_labels(y)
            np.random.seed(self.random_state)

            if feature_names is None:
                feature_names = [f'X_{i + 1}' for i in range(X.shape[1])]
            self.feature_dict_ = get_feature_dict(X.shape[1], feature_names)
            self.feature_placeholders = list(self.feature_dict_.keys())
            self.feature_names = list(self.feature_dict_.values())

            n_train = y.shape[0]
            w = np.ones(n_train) / n_train
            self.estimators_ = []
            self.estimator_weights_ = []
            self.estimator_errors_ = []
            self.estimator_mean_prediction_ = [] # this is just for printing
            if feature_names is not None:
                self.feature_names = feature_names
            for _ in range(self.n_estimators):
                # Fit a classifier with the specific weights
                clf = self.estimator()
                clf.fit(X, y, sample_weight=w)  # uses w as the sampling weight!
                preds = clf.predict(X)
                self.estimator_mean_prediction_.append(np.mean(preds)) # just for printing

                # Indicator function
                miss = preds != y

                # Equivalent with 1/-1 to update weights
                miss2 = np.ones(miss.size)
                miss2[~miss] = -1

                # Error
                err_m = np.dot(w, miss) / sum(w)
                if err_m < 1e-3:
                    return self

                # Alpha
                alpha_m = 0.5 * np.log((1 - err_m) / float(err_m))

                # New weights
                w = np.multiply(w, np.exp([float(x) * alpha_m
                                           for x in miss2]))

                self.estimators_.append(deepcopy(clf))
                self.estimator_weights_.append(alpha_m)
                self.estimator_errors_.append(err_m)

            rules = []

            for est, est_weight in zip(self.estimators_, self.estimator_weights_):
                if type(clf) == DecisionTreeClassifier:
                    est_rules_values = tree_to_rules(est, self.feature_placeholders, prediction_values=True)
                    est_rules = list(map(lambda x: x[0], est_rules_values))

                    # BRS scores are difference between class 1 % and class 0 % in a node
                    est_values = np.array(list(map(lambda x: x[1], est_rules_values)))
                    rule_scores = (est_values[:, 1] - est_values[:, 0]) / est_values.sum(axis=1)

                    compos_score = est_weight * rule_scores
                    rules += [Rule(r, args=[w]) for (r, w) in zip(est_rules, compos_score)]

                if type(clf) == SlipperBaseEstimator:
                    # SLIPPER uses uniform confidence over in rule observations
                    est_rule = dict_to_rule(est.rule, est.feature_dict)
                    rules += [Rule(est_rule, args=[est_weight])]

            self.rules_without_feature_names_ = rules
            self.rules_ = [
                replace_feature_name(rule, self.feature_dict_)
                for rule in self.rules_without_feature_names_
            ]
            self.complexity_ = self._get_complexity()
            return self

        def predict_proba(self, X):
            ''' Predict probabilities for X '''
            check_is_fitted(self)
            X = check_array(X)

            # Add to prediction
            n_train = X.shape[0]
            n_estimators = len(self.estimators_)
            n_classes = 2  # hard-coded for now!
            preds = np.zeros((n_train, n_classes))
            for i in range(n_estimators):
                preds += self.estimator_weights_[i] * self.estimators_[i].predict_proba(X)
            pred_values = preds / n_estimators
            return normalize(pred_values, norm='l1')

        def predict(self, X):
            """Predict outcome for X
            """
            check_is_fitted(self)
            X = check_array(X)
            return self.eval_weighted_rule_sum(X) > 0

        def __str__(self):
            try:
                s = 'BoostedRules:\n'
                s += 'Rule \u2192 predicted probability (final prediction is weighted sum of all predictions)\n'
                for i in range(len(self.estimators_)):
                    s += f'  If\033[96m {str(self.rules_[i])}\033[00m \u2192 {self.estimator_mean_prediction_[i]:.2f} (weight: {self.estimator_weights_[i]:.2f})\n'
                # for est in self.estimators_:
                #     s += '\t' + tree_to_code(est, self.feature_names)
                return s
            except:
                return f'BoostedRules with {len(self.estimators_)} estimators'

## Classes

`class BoostedRulesClassifier (n_estimators=10, estimator=functools.partial(<class 'sklearn.tree._classes.DecisionTreeClassifier'>, max_depth=1), random_state=0)`

  An easy-interpretable classifier optimizing simple logical rules. Currently limited to only binary classification.

  ## Params

  estimator: object with fit and predict methods Defaults to DecisionTreeClassifier with AdaBoost. For SLIPPER, should pass estimator=imodels.SlipperBaseEstimator

  Expand source code

      class BoostedRulesClassifier(RuleSet, BaseEstimator, MetaEstimatorMixin, ClassifierMixin):
          '''An easy-interpretable classifier optimizing simple logical rules.
          Currently limited to only binary classification.

          Params
          ------
          estimator: object with fit and predict methods
              Defaults to DecisionTreeClassifier with AdaBoost.
              For SLIPPER, should pass estimator=imodels.SlipperBaseEstimator
          '''

          def __init__(self, n_estimators=10, estimator=partial(DecisionTreeClassifier, max_depth=1), random_state=0):
              self.n_estimators = n_estimators
              self.estimator = estimator
              self.random_state = random_state

          def fit(self, X, y, feature_names=None, sample_weight=None):
              """Fit the model according to the given training data.

              Parameters
              ----------
              X : array-like, shape (n_samples, n_features)
                  Training vector, where n_samples is the number of samples and
                  n_features is the number of features.

              y : array-like, shape (n_samples,)
                  Target vector relative to X. Has to follow the convention 0 for
                  normal data, 1 for anomalies.

              sample_weight : array-like, shape (n_samples,) optional
                  Array of weights that are assigned to individual samples, typically
                  the amount in case of transactions data. Used to grow regression
                  trees producing further rules to be tested.
                  If not provided, then each sample is given unit weight.

              Returns
              -------
              self : object
                  Returns self.
              """

              X, y = check_X_y(X, y)
              check_classification_targets(y)
              self.n_features_ = X.shape[1]
              self.classes_ = unique_labels(y)
              np.random.seed(self.random_state)

              if feature_names is None:
                  feature_names = [f'X_{i + 1}' for i in range(X.shape[1])]
              self.feature_dict_ = get_feature_dict(X.shape[1], feature_names)
              self.feature_placeholders = list(self.feature_dict_.keys())
              self.feature_names = list(self.feature_dict_.values())

              n_train = y.shape[0]
              w = np.ones(n_train) / n_train
              self.estimators_ = []
              self.estimator_weights_ = []
              self.estimator_errors_ = []
              self.estimator_mean_prediction_ = [] # this is just for printing
              if feature_names is not None:
                  self.feature_names = feature_names
              for _ in range(self.n_estimators):
                  # Fit a classifier with the specific weights
                  clf = self.estimator()
                  clf.fit(X, y, sample_weight=w)  # uses w as the sampling weight!
                  preds = clf.predict(X)
                  self.estimator_mean_prediction_.append(np.mean(preds)) # just for printing

                  # Indicator function
                  miss = preds != y

                  # Equivalent with 1/-1 to update weights
                  miss2 = np.ones(miss.size)
                  miss2[~miss] = -1

                  # Error
                  err_m = np.dot(w, miss) / sum(w)
                  if err_m < 1e-3:
                      return self

                  # Alpha
                  alpha_m = 0.5 * np.log((1 - err_m) / float(err_m))

                  # New weights
                  w = np.multiply(w, np.exp([float(x) * alpha_m
                                             for x in miss2]))

                  self.estimators_.append(deepcopy(clf))
                  self.estimator_weights_.append(alpha_m)
                  self.estimator_errors_.append(err_m)

              rules = []

              for est, est_weight in zip(self.estimators_, self.estimator_weights_):
                  if type(clf) == DecisionTreeClassifier:
                      est_rules_values = tree_to_rules(est, self.feature_placeholders, prediction_values=True)
                      est_rules = list(map(lambda x: x[0], est_rules_values))

                      # BRS scores are difference between class 1 % and class 0 % in a node
                      est_values = np.array(list(map(lambda x: x[1], est_rules_values)))
                      rule_scores = (est_values[:, 1] - est_values[:, 0]) / est_values.sum(axis=1)

                      compos_score = est_weight * rule_scores
                      rules += [Rule(r, args=[w]) for (r, w) in zip(est_rules, compos_score)]

                  if type(clf) == SlipperBaseEstimator:
                      # SLIPPER uses uniform confidence over in rule observations
                      est_rule = dict_to_rule(est.rule, est.feature_dict)
                      rules += [Rule(est_rule, args=[est_weight])]

              self.rules_without_feature_names_ = rules
              self.rules_ = [
                  replace_feature_name(rule, self.feature_dict_)
                  for rule in self.rules_without_feature_names_
              ]
              self.complexity_ = self._get_complexity()
              return self

          def predict_proba(self, X):
              ''' Predict probabilities for X '''
              check_is_fitted(self)
              X = check_array(X)

              # Add to prediction
              n_train = X.shape[0]
              n_estimators = len(self.estimators_)
              n_classes = 2  # hard-coded for now!
              preds = np.zeros((n_train, n_classes))
              for i in range(n_estimators):
                  preds += self.estimator_weights_[i] * self.estimators_[i].predict_proba(X)
              pred_values = preds / n_estimators
              return normalize(pred_values, norm='l1')

          def predict(self, X):
              """Predict outcome for X
              """
              check_is_fitted(self)
              X = check_array(X)
              return self.eval_weighted_rule_sum(X) > 0

          def __str__(self):
              try:
                  s = 'BoostedRules:\n'
                  s += 'Rule \u2192 predicted probability (final prediction is weighted sum of all predictions)\n'
                  for i in range(len(self.estimators_)):
                      s += f'  If\033[96m {str(self.rules_[i])}\033[00m \u2192 {self.estimator_mean_prediction_[i]:.2f} (weight: {self.estimator_weights_[i]:.2f})\n'
                  # for est in self.estimators_:
                  #     s += '\t' + tree_to_code(est, self.feature_names)
                  return s
              except:
                  return f'BoostedRules with {len(self.estimators_)} estimators'

  ### Ancestors

  - RuleSet
  - sklearn.base.BaseEstimator
  - sklearn.base.MetaEstimatorMixin
  - sklearn.base.ClassifierMixin

  ### Subclasses

  - SlipperClassifier

  ### Methods

  `def fit(self, X, y, feature_names=None, sample_weight=None)`

    Fit the model according to the given training data.

    ## Parameters

    `X` :`array-like, shape (n_samples, n_features)`
      Training vector, where n_samples is the number of samples and n_features is the number of features.
    `y` :`array-like, shape (n_samples,)`
      Target vector relative to X. Has to follow the convention 0 for normal data, 1 for anomalies.
    `sample_weight` :`array-like, shape (n_samples,) optional`
      Array of weights that are assigned to individual samples, typically the amount in case of transactions data. Used to grow regression trees producing further rules to be tested. If not provided, then each sample is given unit weight.

    ## Returns

    `self` :`object`
      Returns self.
    Expand source code

        def fit(self, X, y, feature_names=None, sample_weight=None):
            """Fit the model according to the given training data.

            Parameters
            ----------
            X : array-like, shape (n_samples, n_features)
                Training vector, where n_samples is the number of samples and
                n_features is the number of features.

            y : array-like, shape (n_samples,)
                Target vector relative to X. Has to follow the convention 0 for
                normal data, 1 for anomalies.

            sample_weight : array-like, shape (n_samples,) optional
                Array of weights that are assigned to individual samples, typically
                the amount in case of transactions data. Used to grow regression
                trees producing further rules to be tested.
                If not provided, then each sample is given unit weight.

            Returns
            -------
            self : object
                Returns self.
            """

            X, y = check_X_y(X, y)
            check_classification_targets(y)
            self.n_features_ = X.shape[1]
            self.classes_ = unique_labels(y)
            np.random.seed(self.random_state)

            if feature_names is None:
                feature_names = [f'X_{i + 1}' for i in range(X.shape[1])]
            self.feature_dict_ = get_feature_dict(X.shape[1], feature_names)
            self.feature_placeholders = list(self.feature_dict_.keys())
            self.feature_names = list(self.feature_dict_.values())

            n_train = y.shape[0]
            w = np.ones(n_train) / n_train
            self.estimators_ = []
            self.estimator_weights_ = []
            self.estimator_errors_ = []
            self.estimator_mean_prediction_ = [] # this is just for printing
            if feature_names is not None:
                self.feature_names = feature_names
            for _ in range(self.n_estimators):
                # Fit a classifier with the specific weights
                clf = self.estimator()
                clf.fit(X, y, sample_weight=w)  # uses w as the sampling weight!
                preds = clf.predict(X)
                self.estimator_mean_prediction_.append(np.mean(preds)) # just for printing

                # Indicator function
                miss = preds != y

                # Equivalent with 1/-1 to update weights
                miss2 = np.ones(miss.size)
                miss2[~miss] = -1

                # Error
                err_m = np.dot(w, miss) / sum(w)
                if err_m < 1e-3:
                    return self

                # Alpha
                alpha_m = 0.5 * np.log((1 - err_m) / float(err_m))

                # New weights
                w = np.multiply(w, np.exp([float(x) * alpha_m
                                           for x in miss2]))

                self.estimators_.append(deepcopy(clf))
                self.estimator_weights_.append(alpha_m)
                self.estimator_errors_.append(err_m)

            rules = []

            for est, est_weight in zip(self.estimators_, self.estimator_weights_):
                if type(clf) == DecisionTreeClassifier:
                    est_rules_values = tree_to_rules(est, self.feature_placeholders, prediction_values=True)
                    est_rules = list(map(lambda x: x[0], est_rules_values))

                    # BRS scores are difference between class 1 % and class 0 % in a node
                    est_values = np.array(list(map(lambda x: x[1], est_rules_values)))
                    rule_scores = (est_values[:, 1] - est_values[:, 0]) / est_values.sum(axis=1)

                    compos_score = est_weight * rule_scores
                    rules += [Rule(r, args=[w]) for (r, w) in zip(est_rules, compos_score)]

                if type(clf) == SlipperBaseEstimator:
                    # SLIPPER uses uniform confidence over in rule observations
                    est_rule = dict_to_rule(est.rule, est.feature_dict)
                    rules += [Rule(est_rule, args=[est_weight])]

            self.rules_without_feature_names_ = rules
            self.rules_ = [
                replace_feature_name(rule, self.feature_dict_)
                for rule in self.rules_without_feature_names_
            ]
            self.complexity_ = self._get_complexity()
            return self

  `def predict(self, X)`

    Predict outcome for X

    Expand source code

        def predict(self, X):
            """Predict outcome for X
            """
            check_is_fitted(self)
            X = check_array(X)
            return self.eval_weighted_rule_sum(X) > 0

  `def predict_proba(self, X)`

    Predict probabilities for X

    Expand source code

        def predict_proba(self, X):
            ''' Predict probabilities for X '''
            check_is_fitted(self)
            X = check_array(X)

            # Add to prediction
            n_train = X.shape[0]
            n_estimators = len(self.estimators_)
            n_classes = 2  # hard-coded for now!
            preds = np.zeros((n_train, n_classes))
            for i in range(n_estimators):
                preds += self.estimator_weights_[i] * self.estimators_[i].predict_proba(X)
            pred_values = preds / n_estimators
            return normalize(pred_values, norm='l1')