imodels.util.data_util API documentation

# Module `imodels.util.data_util`

Expand source code

    import os.path
    from os.path import join as oj
    from typing import Tuple

    import numpy as np
    import pandas as pd
    import requests
    import sklearn.datasets
    from scipy.sparse import issparse
    from sklearn.datasets import fetch_openml

    def define_openml_outcomes(y, data_id: str):
        if data_id == '59':  # ionosphere, positive is "good" class
            y = (y == 'g').astype(int)
        if data_id == '183':  # abalone, need to convert strings to floats
            y = y.astype(float)
        return y

    def clean_feat_names(feature_names):
        # shouldn't start with a digit
        return ['X_' + x if x[0].isdigit()
                else x
                for x in feature_names]

    def clean_features(X):
        if issparse(X):
            X = X.toarray()
        try:
            return X.astype(float)
        except:
            for j in range(X.shape[1]):
                try:
                    X[:, j].astype(float)
                except:
                    # non-numeric get replaced with numerical values
                    classes, X[:, j] = np.unique(X[:, j], return_inverse=True)
        return X.astype(float)

    def get_clean_dataset(dataset: str = None, data_source: str = 'imodels', data_path='data') -> Tuple[
        np.ndarray, np.ndarray, list]:
        """Return

        Parameters
        ----------
        dataset: str
            csv_file path or dataset id if data_source is specified
        data_source: str
            options: 'imodels', 'pmlb', 'sklearn', 'openml', 'synthetic'
            boolean - whether dataset is a pmlb dataset name

        Returns
        -------
        X, y, feature_names
        """
        assert data_source in ['imodels', 'pmlb', 'sklearn', 'openml', 'synthetic'], data_source + ' not correct'
        if data_source == 'imodels':
            if not dataset.endswith('csv'):
                dataset = dataset + '.csv'
            if not os.path.isfile(dataset):
                download_imodels_dataset(dataset, data_path)
            df = pd.read_csv(oj(data_path, 'imodels_data', dataset))
            X, y = df.iloc[:, :-1].values, df.iloc[:, -1].values
            feature_names = df.columns.values[:-1]
            return np.nan_to_num(X.astype('float32')), y, clean_feat_names(feature_names)
        elif data_source == 'pmlb':
            from pmlb import fetch_data
            feature_names = list(
                fetch_data(dataset, return_X_y=False, local_cache_dir=oj(data_path, 'pmlb_data')).columns)
            feature_names.remove('target')
            X, y = fetch_data(dataset, return_X_y=True, local_cache_dir=oj(data_path, 'pmlb_data'))
            if np.unique(y).size == 2:  # if binary classification, ensure that the classes are 0 and 1
                y -= np.min(y)
            return clean_features(X), y, clean_feat_names(feature_names)
        elif data_source == 'sklearn':
            if dataset == 'diabetes':
                data = sklearn.datasets.load_diabetes()
            elif dataset == 'california_housing':
                data = sklearn.datasets.fetch_california_housing(data_home=oj(data_path, 'sklearn_data'))
            return data['data'], data['target'], clean_feat_names(data['feature_names'])
        elif data_source == 'openml':  # note this api might change in newer sklearn - should give dataset-id not name
            data = sklearn.datasets.fetch_openml(data_id=dataset, data_home=oj(data_path, 'openml_data'))
            X, y, feature_names = data['data'], data['target'], clean_feat_names(data['feature_names'])
            if isinstance(X, pd.DataFrame):
                X = X.values
            if isinstance(y, pd.Series):
                y = y.values
            y = define_openml_outcomes(y, dataset)
            return clean_features(X), y, clean_feat_names(feature_names)
        elif data_source == 'synthetic':
            if dataset == 'friedman1':
                X, y = sklearn.datasets.make_friedman1(n_samples=200, n_features=10)
            elif dataset == 'friedman2':
                X, y = sklearn.datasets.make_friedman2(n_samples=200)
            elif dataset == 'friedman3':
                X, y = sklearn.datasets.make_friedman3(n_samples=200)
            return X, y, ['X_' + str(i + 1) for i in range(X.shape[1])]

    def get_openml_dataset(data_id: int) -> pd.DataFrame:
        dataset = fetch_openml(data_id=data_id, as_frame=False)
        X = dataset.data
        if issparse(X):
            X = X.toarray()
        y = (dataset.target == dataset.target[0]).astype(int)
        feature_names = dataset.feature_names

        target_name = dataset.target_names
        if target_name[0].lower() == 'class':
            target_name = [dataset.target[0]]

        X_df = pd.DataFrame(X, columns=feature_names)
        y_df = pd.DataFrame(y, columns=target_name)
        return pd.concat((X_df, y_df), axis=1)

    def download_imodels_dataset(dataset_fname, data_path: str):
        dataset_fname = dataset_fname.split('/')[-1]  # remove anything about the path
        download_path = f'https://raw.githubusercontent.com/csinva/imodels-data/master/data_cleaned/{dataset_fname}'
        r = requests.get(download_path)
        if r.status_code == 404:
            raise Exception(f'404 Error for dataset {dataset_fname} (see valid files at https://github.com/csinva/imodels-data/tree/master/data_cleaned)')

        os.makedirs(oj(data_path, 'imodels_data'), exist_ok=True)
        with open(oj(data_path, 'imodels_data', dataset_fname), 'w') as f:
            f.write(r.text)

## Functions

`def clean_feat_names(feature_names)`
  Expand source code

      def clean_feat_names(feature_names):
          # shouldn't start with a digit
          return ['X_' + x if x[0].isdigit()
                  else x
                  for x in feature_names]

`def clean_features(X)`
  Expand source code

      def clean_features(X):
          if issparse(X):
              X = X.toarray()
          try:
              return X.astype(float)
          except:
              for j in range(X.shape[1]):
                  try:
                      X[:, j].astype(float)
                  except:
                      # non-numeric get replaced with numerical values
                      classes, X[:, j] = np.unique(X[:, j], return_inverse=True)
          return X.astype(float)

`def define_openml_outcomes(y, data_id: str)`
  Expand source code

      def define_openml_outcomes(y, data_id: str):
          if data_id == '59':  # ionosphere, positive is "good" class
              y = (y == 'g').astype(int)
          if data_id == '183':  # abalone, need to convert strings to floats
              y = y.astype(float)
          return y

`def download_imodels_dataset(dataset_fname, data_path: str)`
  Expand source code

      def download_imodels_dataset(dataset_fname, data_path: str):
          dataset_fname = dataset_fname.split('/')[-1]  # remove anything about the path
          download_path = f'https://raw.githubusercontent.com/csinva/imodels-data/master/data_cleaned/{dataset_fname}'
          r = requests.get(download_path)
          if r.status_code == 404:
              raise Exception(f'404 Error for dataset {dataset_fname} (see valid files at https://github.com/csinva/imodels-data/tree/master/data_cleaned)')

          os.makedirs(oj(data_path, 'imodels_data'), exist_ok=True)
          with open(oj(data_path, 'imodels_data', dataset_fname), 'w') as f:
              f.write(r.text)

`def get_clean_dataset(dataset: str = None, data_source: str = 'imodels', data_path='data') ‑> Tuple[numpy.ndarray, numpy.ndarray, list]`

  Return

  ## Parameters

  `dataset` :`str`
    csv_file path or dataset id if data_source is specified
  `data_source` :`str`
    options: 'imodels', 'pmlb', 'sklearn', 'openml', 'synthetic' boolean - whether dataset is a pmlb dataset name

  ## Returns

  `X, y, feature_names`
  Expand source code

      def get_clean_dataset(dataset: str = None, data_source: str = 'imodels', data_path='data') -> Tuple[
          np.ndarray, np.ndarray, list]:
          """Return

          Parameters
          ----------
          dataset: str
              csv_file path or dataset id if data_source is specified
          data_source: str
              options: 'imodels', 'pmlb', 'sklearn', 'openml', 'synthetic'
              boolean - whether dataset is a pmlb dataset name

          Returns
          -------
          X, y, feature_names
          """
          assert data_source in ['imodels', 'pmlb', 'sklearn', 'openml', 'synthetic'], data_source + ' not correct'
          if data_source == 'imodels':
              if not dataset.endswith('csv'):
                  dataset = dataset + '.csv'
              if not os.path.isfile(dataset):
                  download_imodels_dataset(dataset, data_path)
              df = pd.read_csv(oj(data_path, 'imodels_data', dataset))
              X, y = df.iloc[:, :-1].values, df.iloc[:, -1].values
              feature_names = df.columns.values[:-1]
              return np.nan_to_num(X.astype('float32')), y, clean_feat_names(feature_names)
          elif data_source == 'pmlb':
              from pmlb import fetch_data
              feature_names = list(
                  fetch_data(dataset, return_X_y=False, local_cache_dir=oj(data_path, 'pmlb_data')).columns)
              feature_names.remove('target')
              X, y = fetch_data(dataset, return_X_y=True, local_cache_dir=oj(data_path, 'pmlb_data'))
              if np.unique(y).size == 2:  # if binary classification, ensure that the classes are 0 and 1
                  y -= np.min(y)
              return clean_features(X), y, clean_feat_names(feature_names)
          elif data_source == 'sklearn':
              if dataset == 'diabetes':
                  data = sklearn.datasets.load_diabetes()
              elif dataset == 'california_housing':
                  data = sklearn.datasets.fetch_california_housing(data_home=oj(data_path, 'sklearn_data'))
              return data['data'], data['target'], clean_feat_names(data['feature_names'])
          elif data_source == 'openml':  # note this api might change in newer sklearn - should give dataset-id not name
              data = sklearn.datasets.fetch_openml(data_id=dataset, data_home=oj(data_path, 'openml_data'))
              X, y, feature_names = data['data'], data['target'], clean_feat_names(data['feature_names'])
              if isinstance(X, pd.DataFrame):
                  X = X.values
              if isinstance(y, pd.Series):
                  y = y.values
              y = define_openml_outcomes(y, dataset)
              return clean_features(X), y, clean_feat_names(feature_names)
          elif data_source == 'synthetic':
              if dataset == 'friedman1':
                  X, y = sklearn.datasets.make_friedman1(n_samples=200, n_features=10)
              elif dataset == 'friedman2':
                  X, y = sklearn.datasets.make_friedman2(n_samples=200)
              elif dataset == 'friedman3':
                  X, y = sklearn.datasets.make_friedman3(n_samples=200)
              return X, y, ['X_' + str(i + 1) for i in range(X.shape[1])]

`def get_openml_dataset(data_id: int) ‑> pandas.core.frame.DataFrame`
  Expand source code

      def get_openml_dataset(data_id: int) -> pd.DataFrame:
          dataset = fetch_openml(data_id=data_id, as_frame=False)
          X = dataset.data
          if issparse(X):
              X = X.toarray()
          y = (dataset.target == dataset.target[0]).astype(int)
          feature_names = dataset.feature_names

          target_name = dataset.target_names
          if target_name[0].lower() == 'class':
              target_name = [dataset.target[0]]

          X_df = pd.DataFrame(X, columns=feature_names)
          y_df = pd.DataFrame(y, columns=target_name)
          return pd.concat((X_df, y_df), axis=1)