{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/RecoHut-Projects/recohut/blob/master/tutorials/preprocessing/T859611_Preprocessing_RetailRocket_Session_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing RetailRocket Session Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import timezone, datetime, timedelta\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing method (info/org/org_min_date/days_test/slice/buys):org_min_date\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "preprocessing method [\"info\",\"org\",\"org_min_date\",\"days_test\",\"slice\",\"buys\"]\n",
    "    info: just load and show info\n",
    "    org: from gru4rec (last day => test set)\n",
    "    org_min_date: from gru4rec (last day => test set) but from a minimal date onwards\n",
    "    days_test: adapted from gru4rec (last N days => test set)\n",
    "    slice: new (create multiple train-test-combinations with a sliding window approach  \n",
    "    buys: load buys and safe file to prepared\n",
    "'''\n",
    "# METHOD = \"slice\"\n",
    "METHOD = input('Preprocessing method (info/org/org_min_date/days_test/slice/buys):') or 'slice'\n",
    "assert(METHOD in 'info/org/org_min_date/days_test/slice/buys'.split('/')), 'Invalid Preprocessing method.'\n",
    "\n",
    "'''\n",
    "data config (all methods)\n",
    "'''\n",
    "PATH = './retailrocket/'\n",
    "PATH_PROCESSED = './retailrocket/slices/'\n",
    "FILE = 'events'\n",
    "\n",
    "'''\n",
    "org_min_date config\n",
    "'''\n",
    "MIN_DATE = '2015-09-02'\n",
    "\n",
    "'''\n",
    "filtering config (all methods)\n",
    "'''\n",
    "SESSION_LENGTH = 30 * 60 #30 minutes\n",
    "MIN_SESSION_LENGTH = 2\n",
    "MIN_ITEM_SUPPORT = 5\n",
    "MIN_DATE = '2014-04-01'\n",
    "\n",
    "'''\n",
    "days test default config\n",
    "'''\n",
    "DAYS_TEST = 2\n",
    "\n",
    "'''\n",
    "slicing default config\n",
    "'''\n",
    "NUM_SLICES = 5 #offset in days from the first date in the data set\n",
    "DAYS_OFFSET = 0 #number of days the training start date is shifted after creating one slice\n",
    "DAYS_SHIFT = 27\n",
    "#each slice consists of...\n",
    "DAYS_TRAIN = 25\n",
    "DAYS_TEST = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "retailrocket.zip    100%[===================>]  32.00M   149MB/s    in 0.2s    \n",
      "Archive:  retailrocket.zip\n",
      "   creating: retailrocket/\n",
      "  inflating: retailrocket/events.csv  \n",
      "   creating: retailrocket/prepared_window/\n",
      "  inflating: retailrocket/prepared_window/events.0.hdf  \n",
      "  inflating: retailrocket/prepared_window/events.1.hdf  \n",
      "  inflating: retailrocket/prepared_window/events.2.hdf  \n",
      "  inflating: retailrocket/prepared_window/events.3.hdf  \n",
      "  inflating: retailrocket/prepared_window/events.4.hdf  \n"
     ]
    }
   ],
   "source": [
    "!wget -q --show-progress https://github.com/RecoHut-Datasets/retail_rocket/raw/v2/retailrocket.zip\n",
    "!unzip retailrocket.zip\n",
    "!mkdir retailrocket/slices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing from original gru4rec\n",
    "def preprocess_org( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    split_data_org( data, path_proc+file )\n",
    "\n",
    "#preprocessing from original gru4rec but from a certain point in time\n",
    "def preprocess_org_min_date( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH, min_date=MIN_DATE ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    data = filter_min_date( data, min_date )\n",
    "    split_data_org( data, path_proc+file )\n",
    "\n",
    "#preprocessing adapted from original gru4rec\n",
    "def preprocess_days_test( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH, days_test=DAYS_TEST ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    split_data( data, path_proc+file, days_test )\n",
    "\n",
    "#preprocessing from original gru4rec but from a certain point in time\n",
    "def preprocess_days_test_min_date( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH, days_test=DAYS_TEST, min_date=MIN_DATE ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    data = filter_min_date( data, min_date )\n",
    "    split_data( data, path_proc+file, days_test )\n",
    "\n",
    "#preprocessing to create data slices with a sliding window\n",
    "def preprocess_slices( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH,\n",
    "                       num_slices = NUM_SLICES, days_offset = DAYS_OFFSET, days_shift = DAYS_SHIFT, days_train = DAYS_TRAIN, days_test=DAYS_TEST ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    slice_data( data, path_proc+file, num_slices, days_offset, days_shift, days_train, days_test )\n",
    "    \n",
    "#just load and show info\n",
    "def preprocess_info( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    \n",
    "def preprocess_save( path=PATH, file=FILE, path_proc=PATH_PROCESSED, min_item_support=MIN_ITEM_SUPPORT, min_session_length=MIN_SESSION_LENGTH ):\n",
    "    \n",
    "    data, buys = load_data( path+file )\n",
    "    data = filter_data( data, min_item_support, min_session_length )\n",
    "    data.to_csv(path_proc + file + '_preprocessed.txt', sep='\\t', index=False)\n",
    "    \n",
    "#preprocessing to create a file with buy actions\n",
    "def preprocess_buys( path=PATH, file=FILE, path_proc=PATH_PROCESSED ): \n",
    "    data, buys = load_data( path+file )\n",
    "    store_buys(buys, path_proc+file)\n",
    "    \n",
    "def load_data( file ) : \n",
    "    \n",
    "    #load csv\n",
    "    data = pd.read_csv( file+'.csv', sep=',', header=0, usecols=[0,1,2,3], dtype={0:np.int64, 1:np.int32, 2:str, 3:np.int32})\n",
    "    #specify header names\n",
    "    data.columns = ['Time','UserId','Type','ItemId']\n",
    "    data['Time'] = (data.Time / 1000).astype( int )\n",
    "    \n",
    "    data.sort_values( ['UserId','Time'], ascending=True, inplace=True )\n",
    "    \n",
    "    #sessionize    \n",
    "    data['TimeTmp'] = pd.to_datetime(data.Time, unit='s')\n",
    "    \n",
    "    data.sort_values( ['UserId','TimeTmp'], ascending=True, inplace=True )\n",
    "#     users = data.groupby('UserId')\n",
    "    \n",
    "    data['TimeShift'] = data['TimeTmp'].shift(1)\n",
    "    data['TimeDiff'] = (data['TimeTmp'] - data['TimeShift']).dt.total_seconds().abs()\n",
    "    data['SessionIdTmp'] = (data['TimeDiff'] > SESSION_LENGTH).astype( int )\n",
    "    data['SessionId'] = data['SessionIdTmp'].cumsum( skipna=False )\n",
    "    del data['SessionIdTmp'], data['TimeShift'], data['TimeDiff']\n",
    "    \n",
    "    \n",
    "    data.sort_values( ['SessionId','Time'], ascending=True, inplace=True )\n",
    "    \n",
    "    cart = data[data.Type == 'addtocart']\n",
    "    data = data[data.Type == 'view']\n",
    "    del data['Type']\n",
    "    \n",
    "    print(data)\n",
    "    \n",
    "    #output\n",
    "    \n",
    "    print( data.Time.min() )\n",
    "    print( data.Time.max() )\n",
    "    data_start = datetime.fromtimestamp( data.Time.min(), timezone.utc )\n",
    "    data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )\n",
    "    \n",
    "    del data['TimeTmp']\n",
    "    \n",
    "    print('Loaded data set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}\\n\\n'.\n",
    "          format( len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat() ) )\n",
    "    \n",
    "    return data, cart;\n",
    "\n",
    "\n",
    "def filter_data( data, min_item_support, min_session_length ) : \n",
    "    \n",
    "    #y?\n",
    "    session_lengths = data.groupby('SessionId').size()\n",
    "    data = data[np.in1d(data.SessionId, session_lengths[ session_lengths>1 ].index)]\n",
    "    \n",
    "    #filter item support\n",
    "    item_supports = data.groupby('ItemId').size()\n",
    "    data = data[np.in1d(data.ItemId, item_supports[ item_supports>= min_item_support ].index)]\n",
    "    \n",
    "    #filter session length\n",
    "    session_lengths = data.groupby('SessionId').size()\n",
    "    data = data[np.in1d(data.SessionId, session_lengths[ session_lengths>= min_session_length ].index)]\n",
    "    \n",
    "    #output\n",
    "    data_start = datetime.fromtimestamp( data.Time.min(), timezone.utc )\n",
    "    data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )\n",
    "    \n",
    "    print('Filtered data set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}\\n\\n'.\n",
    "          format( len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat() ) )\n",
    "    \n",
    "    return data;\n",
    "\n",
    "def filter_min_date( data, min_date='2014-04-01' ) :\n",
    "    \n",
    "    min_datetime = datetime.strptime(min_date + ' 00:00:00', '%Y-%m-%d %H:%M:%S')\n",
    "    \n",
    "    #filter\n",
    "    session_max_times = data.groupby('SessionId').Time.max()\n",
    "    session_keep = session_max_times[ session_max_times > min_datetime.timestamp() ].index\n",
    "    \n",
    "    data = data[ np.in1d(data.SessionId, session_keep) ]\n",
    "    \n",
    "    #output\n",
    "    data_start = datetime.fromtimestamp( data.Time.min(), timezone.utc )\n",
    "    data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )\n",
    "    \n",
    "    print('Filtered data set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}\\n\\n'.\n",
    "          format( len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat() ) )\n",
    "    \n",
    "    return data;\n",
    "\n",
    "\n",
    "\n",
    "def split_data_org( data, output_file ) :\n",
    "    \n",
    "    tmax = data.Time.max()\n",
    "    session_max_times = data.groupby('SessionId').Time.max()\n",
    "    session_train = session_max_times[session_max_times < tmax-86400].index\n",
    "    session_test = session_max_times[session_max_times >= tmax-86400].index\n",
    "    train = data[np.in1d(data.SessionId, session_train)]\n",
    "    test = data[np.in1d(data.SessionId, session_test)]\n",
    "    test = test[np.in1d(test.ItemId, train.ItemId)]\n",
    "    tslength = test.groupby('SessionId').size()\n",
    "    test = test[np.in1d(test.SessionId, tslength[tslength>=2].index)]\n",
    "    print('Full train set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}'.format(len(train), train.SessionId.nunique(), train.ItemId.nunique()))\n",
    "    train.to_csv(output_file + '_train_full.txt', sep='\\t', index=False)\n",
    "    print('Test set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}'.format(len(test), test.SessionId.nunique(), test.ItemId.nunique()))\n",
    "    test.to_csv(output_file + '_test.txt', sep='\\t', index=False)\n",
    "    \n",
    "    tmax = train.Time.max()\n",
    "    session_max_times = train.groupby('SessionId').Time.max()\n",
    "    session_train = session_max_times[session_max_times < tmax-86400].index\n",
    "    session_valid = session_max_times[session_max_times >= tmax-86400].index\n",
    "    train_tr = train[np.in1d(train.SessionId, session_train)]\n",
    "    valid = train[np.in1d(train.SessionId, session_valid)]\n",
    "    valid = valid[np.in1d(valid.ItemId, train_tr.ItemId)]\n",
    "    tslength = valid.groupby('SessionId').size()\n",
    "    valid = valid[np.in1d(valid.SessionId, tslength[tslength>=2].index)]\n",
    "    print('Train set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}'.format(len(train_tr), train_tr.SessionId.nunique(), train_tr.ItemId.nunique()))\n",
    "    train_tr.to_csv( output_file + '_train_tr.txt', sep='\\t', index=False)\n",
    "    print('Validation set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}'.format(len(valid), valid.SessionId.nunique(), valid.ItemId.nunique()))\n",
    "    valid.to_csv( output_file + '_train_valid.txt', sep='\\t', index=False)\n",
    "    \n",
    "    \n",
    "    \n",
    "def split_data( data, output_file, days_test ) :\n",
    "    \n",
    "    data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )\n",
    "    test_from = data_end - timedelta( days_test )\n",
    "    \n",
    "    session_max_times = data.groupby('SessionId').Time.max()\n",
    "    session_train = session_max_times[ session_max_times < test_from.timestamp() ].index\n",
    "    session_test = session_max_times[ session_max_times >= test_from.timestamp() ].index\n",
    "    train = data[np.in1d(data.SessionId, session_train)]\n",
    "    test = data[np.in1d(data.SessionId, session_test)]\n",
    "    test = test[np.in1d(test.ItemId, train.ItemId)]\n",
    "    tslength = test.groupby('SessionId').size()\n",
    "    test = test[np.in1d(test.SessionId, tslength[tslength>=2].index)]\n",
    "    print('Full train set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}'.format(len(train), train.SessionId.nunique(), train.ItemId.nunique()))\n",
    "    train.to_csv(output_file + '_train_full.txt', sep='\\t', index=False)\n",
    "    print('Test set\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}'.format(len(test), test.SessionId.nunique(), test.ItemId.nunique()))\n",
    "    test.to_csv(output_file + '_test.txt', sep='\\t', index=False)\n",
    "    \n",
    "    \n",
    "    \n",
    "def slice_data( data, output_file, num_slices, days_offset, days_shift, days_train, days_test ): \n",
    "    \n",
    "    for slice_id in range( 0, num_slices ) :\n",
    "        split_data_slice( data, output_file, slice_id, days_offset+(slice_id*days_shift), days_train, days_test )\n",
    "\n",
    "def split_data_slice( data, output_file, slice_id, days_offset, days_train, days_test ) :\n",
    "    \n",
    "    data_start = datetime.fromtimestamp( data.Time.min(), timezone.utc )\n",
    "    data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )\n",
    "    \n",
    "    print('Full data set {}\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}'.\n",
    "          format( slice_id, len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.isoformat(), data_end.isoformat() ) )\n",
    "    \n",
    "    \n",
    "    start = datetime.fromtimestamp( data.Time.min(), timezone.utc ) + timedelta( days_offset ) \n",
    "    middle =  start + timedelta( days_train )\n",
    "    end =  middle + timedelta( days_test )\n",
    "    \n",
    "    #prefilter the timespan\n",
    "    session_max_times = data.groupby('SessionId').Time.max()\n",
    "    greater_start = session_max_times[session_max_times >= start.timestamp()].index\n",
    "    lower_end = session_max_times[session_max_times <= end.timestamp()].index\n",
    "    data_filtered = data[np.in1d(data.SessionId, greater_start.intersection( lower_end ))]\n",
    "    \n",
    "    print('Slice data set {}\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {} / {}'.\n",
    "          format( slice_id, len(data_filtered), data_filtered.SessionId.nunique(), data_filtered.ItemId.nunique(), start.date().isoformat(), middle.date().isoformat(), end.date().isoformat() ) )\n",
    "    \n",
    "    #split to train and test\n",
    "    session_max_times = data_filtered.groupby('SessionId').Time.max()\n",
    "    sessions_train = session_max_times[session_max_times < middle.timestamp()].index\n",
    "    sessions_test = session_max_times[session_max_times >= middle.timestamp()].index\n",
    "    \n",
    "    train = data[np.in1d(data.SessionId, sessions_train)]\n",
    "    \n",
    "    print('Train set {}\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {}'.\n",
    "          format( slice_id, len(train), train.SessionId.nunique(), train.ItemId.nunique(), start.date().isoformat(), middle.date().isoformat() ) )\n",
    "    \n",
    "    train.to_csv(output_file + '_train_full.'+str(slice_id)+'.txt', sep='\\t', index=False)\n",
    "    \n",
    "    test = data[np.in1d(data.SessionId, sessions_test)]\n",
    "    test = test[np.in1d(test.ItemId, train.ItemId)]\n",
    "    \n",
    "    tslength = test.groupby('SessionId').size()\n",
    "    test = test[np.in1d(test.SessionId, tslength[tslength>=2].index)]\n",
    "    \n",
    "    print('Test set {}\\n\\tEvents: {}\\n\\tSessions: {}\\n\\tItems: {}\\n\\tSpan: {} / {} \\n\\n'.\n",
    "          format( slice_id, len(test), test.SessionId.nunique(), test.ItemId.nunique(), middle.date().isoformat(), end.date().isoformat() ) )\n",
    "    \n",
    "    test.to_csv(output_file + '_test.'+str(slice_id)+'.txt', sep='\\t', index=False)\n",
    "\n",
    "\n",
    "def store_buys( buys, target ):\n",
    "    buys.to_csv( target + '_buys.txt', sep='\\t', index=False )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "START preprocessing  org_min_date\n",
      "               Time   UserId  ItemId             TimeTmp  SessionId\n",
      "1361687  1442004589        0  285930 2015-09-11 20:49:49          0\n",
      "1367212  1442004759        0  357564 2015-09-11 20:52:39          0\n",
      "1367342  1442004917        0   67045 2015-09-11 20:55:17          0\n",
      "830385   1439487966        1   72028 2015-08-13 17:46:06          1\n",
      "742616   1438969904        2  325215 2015-08-07 17:51:44          2\n",
      "...             ...      ...     ...                 ...        ...\n",
      "206556   1433972768  1407575  121220 2015-06-10 21:46:08    1761093\n",
      "47311    1433343689  1407576  356208 2015-06-03 15:01:29    1761094\n",
      "1762583  1431899284  1407577  427784 2015-05-17 21:48:04    1761095\n",
      "1744277  1431825683  1407578  188736 2015-05-17 01:21:23    1761096\n",
      "482559   1435184526  1407579    2521 2015-06-24 22:22:06    1761097\n",
      "\n",
      "[2664312 rows x 5 columns]\n",
      "1430622011\n",
      "1442545187\n",
      "Loaded data set\n",
      "\tEvents: 2664312\n",
      "\tSessions: 1755206\n",
      "\tItems: 234838\n",
      "\tSpan: 2015-05-03 / 2015-09-18\n",
      "\n",
      "\n",
      "Filtered data set\n",
      "\tEvents: 1085763\n",
      "\tSessions: 306919\n",
      "\tItems: 49070\n",
      "\tSpan: 2015-05-03 / 2015-09-18\n",
      "\n",
      "\n",
      "Filtered data set\n",
      "\tEvents: 1085763\n",
      "\tSessions: 306919\n",
      "\tItems: 49070\n",
      "\tSpan: 2015-05-03 / 2015-09-18\n",
      "\n",
      "\n",
      "Full train set\n",
      "\tEvents: 1082094\n",
      "\tSessions: 305845\n",
      "\tItems: 49062\n",
      "Test set\n",
      "\tEvents: 3627\n",
      "\tSessions: 1065\n",
      "\tItems: 2190\n",
      "Train set\n",
      "\tEvents: 1077876\n",
      "\tSessions: 304681\n",
      "\tItems: 49058\n",
      "Validation set\n",
      "\tEvents: 4194\n",
      "\tSessions: 1162\n",
      "\tItems: 2606\n",
      "END preproccessing  16.6654531955719 c  16.66545534133911 s\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    '''\n",
    "    Run the preprocessing configured above.\n",
    "    '''\n",
    "    \n",
    "    print( \"START preprocessing \", METHOD )\n",
    "    sc, st = time.time(), time.time()\n",
    "    \n",
    "    if METHOD == \"info\":\n",
    "        preprocess_info( PATH, FILE, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH )\n",
    "    \n",
    "    elif METHOD == \"org\":\n",
    "        preprocess_org( PATH, FILE, PATH_PROCESSED, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH )\n",
    "     \n",
    "    elif METHOD == \"org_min_date\":\n",
    "        preprocess_org_min_date( PATH, FILE, PATH_PROCESSED, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH, MIN_DATE )\n",
    "        \n",
    "    elif METHOD == \"day_test\":\n",
    "        preprocess_days_test( PATH, FILE, PATH_PROCESSED, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH, DAYS_TEST )\n",
    "        \n",
    "    elif METHOD == \"day_test_min_date\":\n",
    "        preprocess_days_test_min_date( PATH, FILE, PATH_PROCESSED, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH, DAYS_TEST, MIN_DATE )\n",
    "    \n",
    "    elif METHOD == \"slice\":\n",
    "        preprocess_slices( PATH, FILE, PATH_PROCESSED, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH, NUM_SLICES, DAYS_OFFSET, DAYS_SHIFT, DAYS_TRAIN, DAYS_TEST )\n",
    "        \n",
    "    elif METHOD == \"buys\":\n",
    "        preprocess_buys( PATH, FILE, PATH_PROCESSED )\n",
    "        \n",
    "    elif METHOD == \"save\":\n",
    "        preprocess_save( PATH, FILE, PATH_PROCESSED, MIN_ITEM_SUPPORT, MIN_SESSION_LENGTH )\n",
    "        \n",
    "    else: \n",
    "        print( \"Invalid method \", METHOD )\n",
    "        \n",
    "    print( \"END preproccessing \", (time.time() - sc), \"c \", (time.time() - st), \"s\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !apt-get -qq install tree\n",
    "# !rm -r sample_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".\n",
      "├── [217M]  retailrocket\n",
      "│   ├── [ 90M]  events.csv\n",
      "│   ├── [ 60M]  prepared_window\n",
      "│   │   ├── [ 12M]  events.0.hdf\n",
      "│   │   ├── [ 12M]  events.1.hdf\n",
      "│   │   ├── [ 12M]  events.2.hdf\n",
      "│   │   ├── [ 11M]  events.3.hdf\n",
      "│   │   └── [ 11M]  events.4.hdf\n",
      "│   └── [ 67M]  slices\n",
      "│       ├── [115K]  events_test.txt\n",
      "│       ├── [ 33M]  events_train_full.txt\n",
      "│       ├── [ 33M]  events_train_tr.txt\n",
      "│       └── [132K]  events_train_valid.txt\n",
      "└── [ 32M]  retailrocket.zip\n",
      "\n",
      " 249M used in 3 directories, 11 files\n"
     ]
    }
   ],
   "source": [
    "# !tree -h --du ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-04 17:20:19\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "IPython: 5.5.0\n",
      "pandas : 1.1.5\n",
      "numpy  : 1.19.5\n",
      "sys    : 3.7.12 (default, Sep 10 2021, 00:21:48) \n",
      "[GCC 7.5.0]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# !pip install -q watermark\n",
    "# %reload_ext watermark\n",
    "# %watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**END**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
