# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/preprocessing.ipynb (unless otherwise specified).

__all__ = ['split_dataset', 'Processor', 'DEP_VAR', 'TIME_COL', 'site_GMT_offsets', 'GMT_offset_map',
           'DEFAULT_GRP_COLS', 'DEFAULT_ONEHOT_COLS', 'align_test', 'test_var_names', 'store_var_names',
           'load_var_names', 'store_df', 'load_df', 'preprocess_all']

# Cell
import pandas as pd
from pathlib import Path
import os
import plotly.graph_objects as go
import plotly.express as px
import numpy as np
import typing
import pickle
import ipywidgets as widgets

from ashrae import loading, inspection

from fastai.tabular.all import *

import tqdm

from sklearn import linear_model, tree, model_selection, ensemble
from sklearn.preprocessing import OneHotEncoder

import itertools
from pandas.tseries.holiday import USFederalHolidayCalendar as us_calendar

import math
from loguru import logger

from scipy import stats

# Cell
def split_dataset(X:pd.DataFrame, split_kind:str='random',
                  train_frac:float=.8, t_train:pd.DataFrame=None):

    def random_split():
        n_train = int(len(X)*train_frac)
        train_bool = X.index.isin(np.random.choice(X.index.values, size=n_train, replace=False))
        return train_bool

    def time_split():
        assert 'timestamp' in X.columns
        time_col = 'timestamp'
        ts = X[time_col].sort_values(ascending=True)
        ix = int(len(X)*train_frac)
        threshold_t = ts.iloc[ix:].values[0]
        return X[time_col] < threshold_t

    def time_split_day():
        time_col = 'timestampDayofyear'

        if time_col not in X.columns:
            t = X['timestamp'].dt.dayofyear
        else:
            t = X[time_col]

        days = (t.value_counts()
                .rename('count')
                .sample(frac=1)
                .to_frame()
                .cumsum()
                .pipe(lambda x: x.loc[x['count'] <= (train_frac * len(t))]))

        num_train_days = len(days)
        mask = t.isin(days.index.values)

        assert mask.sum() > 0
        return mask

    def fix_time_split():
        assert t_train is not None
        time_col = 'timestamp'
        assert time_col in X.columns

        mask = X[time_col].isin(t_train[time_col])
        assert mask.sum() > 0
        return mask

    split_funs = {
        'random': random_split,
        'time': time_split,
        'fix_time': fix_time_split,
        'time_split_day': time_split_day,
    }

    assert split_kind in split_funs
    train_bool = split_funs[split_kind]()

    train_idx = np.where(train_bool)[0]
    valid_idx = np.where(~train_bool)[0]

    return (list(train_idx), list(valid_idx))

# Cell
DEP_VAR = 'meter_reading'
TIME_COL = 'timestamp'

class Processor:

    dep_var_stats:dict = None

    def __init__(self, dep_var:str=None, time_col:str=None,
                 t_train:pd.Series=None):
        self.dep_var = DEP_VAR if dep_var is None else dep_var
        self.time_col = TIME_COL if time_col is None else time_col
        self.t_train = t_train

    def __call__(self, df_core:pd.DataFrame, df_building:pd.DataFrame=None,
                 df_weather:pd.DataFrame=None, t_train:pd.DataFrame=None,
                 tfms_configs:dict=None) -> (pd.DataFrame, dict):

        # check if `df` is a test set (dep_var is missing)
        self.is_train = self.dep_var in df_core.columns
        self.df_core = df_core.copy()
        self.conts, self.cats, self.cats_order = [], [], {}
        self.cats += ['building_id', 'meter']
        self.n = len(df_core)

        # core pieces of dependent and independent variables
        self.update_dep_var()

        if tfms_configs is None:
            logger.info('Empty transform configs `tfms_configs`. Returning ...')
        else:
            # if `t_train` (timestamps which belong to the training set) are provided perform a check which rows are effected
            if self.t_train is not None:
                self.t_train_set = set(self.t_train.values.ravel())

            self.df_building = df_building.copy() if df_building is not None else None
            self.df_weather = df_weather.copy() if df_weather is not None else None

            # running transformations
            self.sanity_check_input_for_tfms(tfms_configs)
            for fun_name, config in tfms_configs.items():
                self.df_core = getattr(self, fun_name)(**config)

        df_core, var_names = self.cleanup()
        logger.info(f'Reduced samples by {self.n - len(df_core)} rows = {(self.n - len(df_core))/self.n*100:.2f} %')
        return df_core, var_names

    @property
    def t_in_train_set(self):
        return self.df_core['timestamp'].isin(self.t_train_set)

    def update_dep_var(self) -> pd.DataFrame:

        if self.dep_var.endswith('log1p'):
            return self.df_core

        dep_var_new = f'{self.dep_var}_log1p'
        if self.is_train:
            self.df_core[dep_var_new] = np.log(self.df_core[self.dep_var].values + 1) # 3s
#             self.df_core[dep_var_new] = self.df_core[self.dep_var].apply(lambda x: math.log(x+1)) # 12s with math.log, 27s with np.log
#             self.df_core[dep_var_new] = self.df_core[self.dep_var].swifter.apply(lambda x: math.log(x+1)) # 15s with math.log + swifter
        self.dep_var = dep_var_new
        return self.df_core

    def sanity_check_input_for_tfms(self, tfms_configs:dict):
        # sanity check presence of df_building if df_weather is given
        if self.df_weather is not None:
            assert self.df_building is not None, 'To join the weather info in `df_weather` you need to pass `df_building`.'

        # making sure all required inputs are specified in `tfms_configs`
        self.test_run = True
        if tfms_configs is not None:
            building_fun_names = ['add_building_features']
            weather_fun_names = ['add_weather_features']
            for fun_name, config in tfms_configs.items():
                getattr(self, fun_name)(**config)
                if fun_name in building_fun_names:
                    assert self.df_building is not None, 'You need to pass `df_building` in Processor.__call__.'
                if fun_name in weather_fun_names:
                    assert self.df_weather is not None, 'You need to pass `df_weather` in Processor.__call__.'
        self.test_run = False

    def get_var_names(self) -> dict:
        return {'conts': self.conts, 'cats': self.cats, 'dep_var': self.dep_var}

    def cleanup(self) -> (pd.DataFrame, dict):
        # converting cats to category type
        for col in self.cats:
            if self.df_core[col].dtype == bool: continue
            self.df_core[col] = self.df_core[col].astype('category')
            if col in self.cats_order:
                self.df_core[col].cat.set_categories(self.cats_order[col],
                                                     ordered=True, inplace=True)

        # removing features
        to_remove_cols = ['meter_reading', 'timestampYear'] # , self.time_col
        self.df_core.drop(columns=[c for c in self.df_core.columns if c in to_remove_cols],
                          inplace=True)

        # shrinking the data frame
        self.df_core = df_shrink(self.df_core, int2uint=True)

        var_names = self.get_var_names()

        if not self.is_train:
            self.df_core.set_index('row_id', inplace=True)

        missing_cols = [col for col in self.df_core.columns.values if col not in self.cats + self.conts + [self.dep_var]
                        and col not in ['timestampElapsed', self.time_col, 'meter_reading']]

        assert len(missing_cols) == 0, f'Missed to assign columns: {missing_cols} to `conts` or `cats`'

        return self.df_core, var_names

# Cell
@patch
def fix_bid_363(self:Processor):
    if self.test_run: return
    if not self.is_train: return self.df_core
    assert 'timestamp' in self.df_core.columns
    rm = (self.df_core['building_id'] == 363)
    rm = rm & (self.df_core['meter'] == 0)
    rm = rm & (self.df_core['timestamp'] < pd.to_datetime('2016-07-30'))
    ok = ~rm
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set
    logger.info(f'Fixing building_id 363: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:]
    return self.df_core

# Cell
@patch
def remove_0s_meter0(self:Processor):
    if self.test_run: return
    if not self.is_train: return self.df_core
    assert 'timestamp' in self.df_core.columns
    rm = (self.df_core['meter'] == 0)
    rm = rm & (self.df_core[self.dep_var] == 0)
    ok = ~rm
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set
    logger.info(f'Removing 0s for meter 0: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:]
    return self.df_core

# Cell
@patch
def remove_not_summer_0s_meter_2_and_3(self:Processor):
    if self.test_run: return
    if not self.is_train: return self.df_core
    assert 'timestamp' in self.df_core.columns
    add_month = 'timestampMonth' not in self.df_core.columns
    if add_month:
        self.df_core['timestampMonth'] = self.df_core['timestamp'].dt.month
    rm = (self.df_core['meter'].isin([2,3]))
    rm = rm & (self.df_core[self.dep_var] == 0)
    rm = rm & (self.df_core['timestampMonth'].isin([6,7,8]))
    ok = ~rm
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set
    logger.info(f'Removing 0s for meter 2 and 3 during summer: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:]
    if add_month:
        self.df_core.drop(columns=['timestampMonth'],inplace=True)
    return self.df_core

# Cell
@patch
def remove_bad_meter0_readings_of_first_141days(self:Processor):
    if self.test_run: return
    if not self.is_train: return self.df_core
    assert 'timestamp' in self.df_core.columns
    add_month = 'timestampDayofyear' not in self.df_core.columns
    if add_month:
        self.df_core['timestampDayofyear'] = self.df_core['timestamp'].dt.dayofyear
    add_site_id = 'site_id' not in self.df_core.columns
    if add_site_id:
        assert self.df_building is not None, 'df_building cannot be None for this method.'
        self.df_core = pd.merge(self.df_core, self.df_building.loc[:,['building_id','site_id']], on='building_id', how='left')
        assert self.df_core['site_id'].isna().sum() == 0
    rm = self.df_core['meter'] == 0
    rm = rm & (self.df_core['site_id'] == 0)
    rm = rm & (self.df_core['timestampDayofyear'] < 141)
    ok = ~rm
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set
    logger.info(f'Bad readings for meter 0 for the first 141 days for site 0: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:]
    if add_month:
        self.df_core.drop(columns=['timestampDayofyear'],inplace=True)
    if add_site_id:
        self.df_core.drop(columns=['site_id'],inplace=True)
    return self.df_core

# Cell
@patch
def fix_bid_1099(self:Processor, threshold:float=10.):
    if self.test_run: return
    if not self.is_train: return self.df_core
    assert 'timestamp' in self.df_core.columns
    rm = (self.df_core['building_id'] == 1099)
    rm = rm & (self.df_core['meter'] == 2)
    rm = rm & (self.df_core[self.dep_var] > threshold)
    ok = ~rm
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set
    logger.info(f'Fixing building_id 1099: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:]
    return self.df_core

# Cell
@patch
def remove_imputed_weeks(self:Processor, dep_var='meter_reading'):
    if self.test_run: return
    if not self.is_train: return self.df_core
    grp = ['building_id', 'meter', pd.Grouper(key='timestamp', freq='W-MON')]
    wks = (self.df_core.groupby(grp)[dep_var]).describe(percentiles=[.05,.95])

    min_date = self.df_core['timestamp'].dt.date.min() - pd.Timedelta(7,unit='w')
    max_date = self.df_core['timestamp'].dt.date.max() + pd.Timedelta(7,unit='d')
    w_range = pd.date_range(min_date, max_date, freq='W-MON')

    self.df_core['week'] = [v.right for v in pd.cut(self.df_core['timestamp'], w_range)]

    self.df_core = self.df_core.join(wks.loc[:,['5%','95%']],
                                     on=['building_id','meter','week'])
    rm = np.isclose(self.df_core['5%'], self.df_core['95%'])
    ok = ~rm
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set
    logger.info(f'Imputed weeks: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:].drop(columns=['5%','95%','week'])
    return self.df_core

# Cell
@patch
def remove_outliers(self:Processor, f:float=10, dep_var:str='meter_reading'):
    if self.test_run: return None
    if not self.is_train: return self.df_core
    s = self.df_core.groupby(['building_id','meter'])[dep_var].describe()
    s['threshold'] = s['50%'] + (s['75%'] - s['50%']) * f
    self.df_core = self.df_core.join(s.loc[:,['threshold']],
                                     on=['building_id', 'meter'])
    ok = self.df_core[dep_var] < self.df_core['threshold']
    if self.is_train and self.t_train is not None:
        ok = ok | ~self.t_in_train_set

    logger.info(f'Outliers: removing {(~ok).sum()} data points = {(~ok).sum()/len(ok)*100:.2f} %')
    self.df_core = self.df_core.loc[ok,:].drop(columns=['threshold'])
    return self.df_core

# Cell
@patch
def add_random_noise_features(self:Processor, col:str='random_noise',
                              noise_func:typing.Callable=lambda x: stats.norm.rvs(size=x, loc=0, scale=1),
                              noise_type:str='cont'):
    assert noise_type in ['cont', 'cat']
    if self.test_run: return
    n = len(self.df_core)
    self.df_core[col] = noise_func(n)

    if noise_type == 'cont':
        self.conts.append(col)
        logger.info(f'Added noise feature: \n\tcontinuous: {[col]}')
    else:
        self.cats.append(col)
        logger.info(f'Added noise feature: \n\tcategorical: {[col]}')

    return self.df_core

# Cell
@patch
def add_building_features(self:Processor):
    if self.test_run: return
    n = len(self.df_core)
    self.df_core = pd.merge(self.df_core, self.df_building, on='building_id', how='left')
    assert n == len(self.df_core)
    _cats = ['site_id', 'primary_use']
    _conts = ['square_feet', 'year_built', 'floor_count']
    logger.info(f'Added building features: \n\tcategorical: {_cats}\n\tcontinuous: {_conts}')
    self.cats.extend(_cats)
    self.conts.extend(_conts)
    return self.df_core

# Cell
site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}

@patch
def add_weather_features(self:Processor,
                         fix_time_offset:bool=False,
                         add_na_indicators:bool=False,
                         impute_nas:bool=False):
    if self.test_run: return
    n = len(self.df_core)
    add_site_id = 'site_id' not in self.df_core.columns
    if add_site_id:
        self.df_core = self.df_core.join(self.df_building.set_index('building_id').loc[:,['site_id']],
                                         on='building_id')

    if fix_time_offset:
        dt = (self.df_weather['site_id']
              .map(GMT_offset_map)
              .apply(lambda x: pd.Timedelta(x, unit='hours')))
        self.df_weather['timestamp'] = self.df_weather['timestamp'] + dt

    cols = ['air_temperature', 'cloud_coverage', 'dew_temperature',
            'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',
            'wind_speed']

    # adding na columns
    na_cols = []
    if add_na_indicators:
        for col in cols:
            nas = self.df_weather[col].isna()
            na_col = f'{col}_na(processor)'
            if nas.sum()>0:
                self.df_weather[na_col] = nas
                self.df_weather[na_col] = self.df_weather[na_col].astype(bool)
                na_cols.append(na_col)

    # imputing na columns
    if impute_nas:
        new_weather = []
        aggs = {col: self.df_weather[col].median()
                for col in cols}
        for site, grp in self.df_weather.groupby('site_id'):
            grp = grp.sort_values('timestamp')
            for col in cols:
                nas = grp[col].isna()
                if nas.sum() == len(grp):
                    grp[col] = aggs[col]
                elif nas.sum() > 0:
                    grp[col] = grp[col].interpolate(limit_direction='both',
                                                    method='linear')
                nas = grp[col].isna()
                grp.loc[nas, col] = aggs[col]

            new_weather.append(grp)
        n_weather = len(self.df_weather)
        self.df_weather = pd.concat(new_weather)
        assert len(self.df_weather) == n_weather, f'Interpolation step changed rows from {n_weather} to {len(self.df_weather)}'

    self.df_core = pd.merge(self.df_core, self.df_weather,
                            on=['site_id', 'timestamp'],
                            how='left')
    assert n == len(self.df_core), f'Merging lead to an increase from {n} rows to {len(self.df_core)}'

    if add_site_id:
        self.df_core.drop(columns=['site_id'], inplace=True)

    _conts = ['wind_direction', 'air_temperature', 'dew_temperature', 'precip_depth_1_hr',
              'sea_level_pressure', 'wind_speed', 'cloud_coverage']
    self.cats.extend(na_cols)
#     self.cats_order['cloud_coverage'] = sorted([v for v in self.df_core['cloud_coverage'].unique() if np.isfinite(v)])
    self.conts.extend(_conts)
    logger.info(f'Added weather features: \n\tcategorical: {na_cols}\n\tcontinuous: {_conts}')
    return self.df_core

# Cell
@patch
def add_time_features(self:Processor):
    if self.test_run: return
    _cats = ['timestampMonth', 'timestampDay', 'timestampWeek', 'timestampDayofweek',
                      'timestampDayofyear', 'timestampIs_month_end', 'timestampIs_month_start',
                      'timestampIs_quarter_start', 'timestampIs_quarter_end',
                      'timestampIs_year_start', 'timestampIs_year_end', 'timestampHour',
                      'timestampIs_us_holiday']
    self.cats.extend(_cats)

    self.df_core = add_datepart(self.df_core, self.time_col, drop=False)

    self.df_core['timestampHour'] = self.df_core[self.time_col].dt.hour

    dates_range = pd.date_range(start='2015-12-31', end='2019-01-01')
    us_holidays = us_calendar().holidays(start=dates_range.min(), end=dates_range.max())

    self.df_core['timestampIs_us_holiday'] = (self.df_core['timestamp'].dt.date.astype('datetime64')
                                              .isin(us_holidays)
                                              .astype(bool))
    logger.info(f'Added categorical time features: {_cats}')
    self.cats_order.update({
        c: sorted(self.df_core[c].unique()) for c in ['timestampMonth', 'timestampDay',
                                                      'timestampWeek', 'timestampDayofweek',
                                                      'timestampDayofyear', 'timestampHour']
    })
    return self.df_core

# Cell
DEFAULT_GRP_COLS = ['building_id', 'timestampHour', 'meter']

@patch
def add_dep_var_stats(self:Processor, grp_cols:typing.List[str]=None):
    if self.test_run: return

    grp_cols = DEFAULT_GRP_COLS if grp_cols is None else grp_cols

    assert self.is_train or self.dep_var_stats is not None
    if self.is_train:
        self.dep_var_stats = dict()

    funs = {
        'median': lambda x: torch.median(tensor(x)).item(),
        'mean': lambda x: torch.mean(tensor(x)).item(),
        '5%': lambda x: np.percentile(x, 5),
        '95%': lambda x: np.percentile(x, 95),
    }
    _conts = []
    # computing stats for self.dep_var on the coarsest possible level
    for name, fun in funs.items():
        name = f'{self.dep_var}_{name}'
        _conts.append(name)
        self.conts.append(name)

        if self.is_train:
            value = fun(self.df_core[self.dep_var].values)
            self.df_core[name] = value
            self.dep_var_stats[name] = value
        else:
            self.df_core[name] = self.dep_var_stats[name]

    # adding stats of self.dep_var on a more granular level
    if grp_cols is not None:
        t_col = 'timestampHour'
        do_add_t = t_col in grp_cols and t_col not in self.df_core.columns.values
        if do_add_t:
            self.df_core[t_col] = self.df_core['timestamp'].dt.hour

        assert all([c in self.df_core.columns.values for c in grp_cols])

        for fun_name, fun in funs.items():
            name = f'{self.dep_var}_{"-".join(grp_cols)}_{fun_name}'
            _conts.append(name)
            self.conts.append(name)

            if self.is_train:

                self.dep_var_stats[name] = (self.df_core.groupby(grp_cols)[self.dep_var]
                                            .agg(fun)
                                            .rename(name))
            self.df_core = self.df_core.join(self.dep_var_stats[name], on=grp_cols)
            self.df_core[name].fillna(self.dep_var_stats[f'{self.dep_var}_{fun_name}'], inplace=True)

        if do_add_t:
            self.df_core.drop(columns=[t_col], inplace=True)
    logger.info(f'Added continuous target columns: {_conts}')
    return self.df_core

# Cell

DEFAULT_ONEHOT_COLS = ['meter']

@patch
def add_onehot_encoded(self:Processor, onehot_cols:typing.List[str]=None):
    if self.test_run: return
    onehot_cols = DEFAULT_ONEHOT_COLS if onehot_cols is None else onehot_cols

    t_col = 'timestampHour'
    do_add_t = t_col in onehot_cols and t_col not in self.df_core.columns.values
    if do_add_t:
        self.df_core[t_col] = self.df_core['timestamp'].dt.hour

    self.df_core['id'] = [str(v) for v in zip(*[self.df_core[v] for v in onehot_cols])]

    if self.is_train:
        self.onehot_cols = onehot_cols
        self.onehot_tfm = OneHotEncoder()
        self.onehot_tfm.fit(self.df_core.loc[:, ['id']])


    names = [f'{"-".join(self.onehot_cols)}_{v}' for v in self.onehot_tfm.categories_[0]]

    self.cats.extend(names)

    df_onehot = pd.DataFrame(self.onehot_tfm.transform(self.df_core.loc[:, ['id']]).toarray(),
                             columns=names, index=self.df_core.index, dtype=bool)
    logger.info(f'Added one hot encoded features: {names}')
    to_drop = ['id']
    if do_add_t:
        to_drop.append(t_col)
    self.df_core.drop(columns=to_drop, inplace=True)
    self.df_core = pd.concat((self.df_core, df_onehot), axis=1)
    return self.df_core

# Cell
def align_test(df_train:pd.DataFrame, var_names:dict,
               df_test:pd.DataFrame):
    return df_test.loc[:,[v for v in df_train.columns if v != var_names['dep_var']]]

# Cell
def test_var_names(var_names:dict):
    assert isinstance(var_names, dict)
    assert 'conts' in var_names and 'cats' in var_names and 'dep_var' in var_names
    assert isinstance(var_names['conts'], list)
    assert isinstance(var_names['cats'], list)
    assert isinstance(var_names['dep_var'], str)

# Cell
def store_var_names(data_path:Path, var_names:dict):
    fname = data_path/'var_names.pckl'
    print(f'Storing var names at: {fname}')
    with open(fname, 'wb') as f:
        pickle.dump(var_names, f)

# Cell
def load_var_names(fname:Path):
    print(f'Reading var names at: {fname}')
    with open(fname, 'rb') as f:
        var_names = pickle.load(f)
    return var_names

# Cell
def store_df(path:Path, df:pd.DataFrame): df.to_parquet(path)

# Cell
def load_df(path:Path): return pd.read_parquet(path)

# Cell
@typed
def preprocess_all(ashrae_data:dict, tfms_config:dict):

    processor = Processor() # t_train=t_train

    df, var_names = processor(ashrae_data['meter_train'], tfms_configs=tfms_config,
                              df_weather=ashrae_data['weather_train'],
                              df_building=ashrae_data['building'])

    df_test_p, _ = processor(ashrae_data['meter_test'], tfms_configs=tfms_config,
                             df_weather=ashrae_data['weather_test'],
                             df_building=ashrae_data['building'])

    test_nans = loading.show_nans(df_test_p)
    train_nans = loading.show_nans(df)

    test_nan_cols = [col for col in test_nans.loc[test_nans['# NaNs']>0].index]
    assert (train_nans.loc[train_nans.index.isin(test_nan_cols),'# NaNs'] == 0).sum() == 0

    df_test = align_test(df, var_names, df_test_p)
    assert len(df_test) == len(ashrae_data['meter_test'])

    assert len(df_test.columns) + 1 == len(df.columns)

    test_var_names(var_names)

    return df, df_test, var_names