{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp transforms.datasets.criteo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Criteo Dataset Transformation\n",
    "> Implementation of transformation functions specific to criteo ad-display dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder, KBinsDiscretizer\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def sparseFeature(feat, feat_num, embed_dim=4):\n",
    "    \"\"\"\n",
    "    create dictionary for sparse feature\n",
    "    :param feat: feature name\n",
    "    :param feat_num: the total number of sparse features that do not repeat\n",
    "    :param embed_dim: embedding dimension\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    return {'feat_name': feat, 'feat_num': feat_num, 'embed_dim': embed_dim}\n",
    "\n",
    "\n",
    "def denseFeature(feat):\n",
    "    \"\"\"\n",
    "    create dictionary for dense feature\n",
    "    :param feat: dense feature name\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    return {'feat_name': feat}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def create_criteo_dataset(file, embed_dim=8, read_part=True, sample_num=100000, test_size=0.2):\n",
    "    \"\"\"\n",
    "    a example about creating criteo dataset\n",
    "    :param file: dataset's path\n",
    "    :param embed_dim: the embedding dimension of sparse features\n",
    "    :param read_part: whether to read part of it\n",
    "    :param sample_num: the number of instances if read_part is True\n",
    "    :param test_size: ratio of test dataset\n",
    "    :return: feature columns, train, test\n",
    "    \"\"\"\n",
    "    names = ['label', 'I1', 'I2', 'I3', 'I4', 'I5', 'I6', 'I7', 'I8', 'I9', 'I10', 'I11',\n",
    "             'I12', 'I13', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',\n",
    "             'C12', 'C13', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21', 'C22',\n",
    "             'C23', 'C24', 'C25', 'C26']\n",
    "\n",
    "    if read_part:\n",
    "        data_df = pd.read_csv(file, sep='\\t', iterator=True, header=None,\n",
    "                          names=names)\n",
    "        data_df = data_df.get_chunk(sample_num)\n",
    "\n",
    "    else:\n",
    "        data_df = pd.read_csv(file, sep='\\t', header=None, names=names)\n",
    "\n",
    "    sparse_features = ['C' + str(i) for i in range(1, 27)]\n",
    "    dense_features = ['I' + str(i) for i in range(1, 14)]\n",
    "    features = sparse_features + dense_features\n",
    "\n",
    "    data_df[sparse_features] = data_df[sparse_features].fillna('-1')\n",
    "    data_df[dense_features] = data_df[dense_features].fillna(0)\n",
    "\n",
    "    # Bin continuous data into intervals.\n",
    "    est = KBinsDiscretizer(n_bins=100, encode='ordinal', strategy='uniform')\n",
    "    data_df[dense_features] = est.fit_transform(data_df[dense_features])\n",
    "\n",
    "    for feat in sparse_features:\n",
    "        le = LabelEncoder()\n",
    "        data_df[feat] = le.fit_transform(data_df[feat])\n",
    "\n",
    "    # ==============Feature Engineering===================\n",
    "\n",
    "    # ====================================================\n",
    "    feature_columns = [sparseFeature(feat, int(data_df[feat].max()) + 1, embed_dim=embed_dim)\n",
    "                        for feat in features]\n",
    "    train, test = train_test_split(data_df, test_size=test_size)\n",
    "\n",
    "    train_X = train[features].values.astype('int32')\n",
    "    train_y = train['label'].values.astype('int32')\n",
    "    test_X = test[features].values.astype('int32')\n",
    "    test_y = test['label'].values.astype('int32')\n",
    "\n",
    "    return feature_columns, (train_X, train_y), (test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting kaggle\n",
      "  Downloading kaggle-1.5.12.tar.gz (58 kB)\n",
      "\u001b[?25l\r\u001b[K     |█████▋                          | 10 kB 19.0 MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 20 kB 20.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 30 kB 14.2 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 40 kB 10.5 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 51 kB 4.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 58 kB 2.7 MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: kaggle\n",
      "  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for kaggle: filename=kaggle-1.5.12-py3-none-any.whl size=73051 sha256=a09d2576937c68b6341e6bce9eeefa020563e125d97e69548f4d591568008b5f\n",
      "  Stored in directory: /root/.cache/pip/wheels/62/d6/58/5853130f941e75b2177d281eb7e44b4a98ed46dd155f556dc5\n",
      "Successfully built kaggle\n",
      "Installing collected packages: kaggle\n",
      "  Attempting uninstall: kaggle\n",
      "    Found existing installation: kaggle 1.5.12\n",
      "    Uninstalling kaggle-1.5.12:\n",
      "      Successfully uninstalled kaggle-1.5.12\n",
      "Successfully installed kaggle-1.5.12\n",
      "Downloading criteo-dataset.zip to /content\n",
      "100% 4.31G/4.31G [01:20<00:00, 58.4MB/s]\n",
      "100% 4.31G/4.31G [01:20<00:00, 57.4MB/s]\n",
      "Archive:  criteo-dataset.zip\n",
      "  inflating: dac/readme.txt          \n",
      "  inflating: dac/test.txt            \n",
      "  inflating: dac/train.txt           \n"
     ]
    }
   ],
   "source": [
    "# !pip install -q -U kaggle\n",
    "# !pip install --upgrade --force-reinstall --no-deps kaggle\n",
    "# !mkdir ~/.kaggle\n",
    "# !cp /content/drive/MyDrive/kaggle.json ~/.kaggle/\n",
    "# !chmod 600 ~/.kaggle/kaggle.json\n",
    "# !kaggle datasets download -d mrkmakr/criteo-dataset\n",
    "# !unzip criteo-dataset.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'dac/train.txt'\n",
    "read_part = True\n",
    "sample_num = 10000\n",
    "test_size = 0.2\n",
    "\n",
    "feature_columns, train, test = create_criteo_dataset(file=file,\n",
    "                                        read_part=read_part,\n",
    "                                        sample_num=sample_num,\n",
    "                                        test_size=test_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'embed_dim': 8, 'feat_name': 'C1', 'feat_num': 175},\n",
       " {'embed_dim': 8, 'feat_name': 'C2', 'feat_num': 386},\n",
       " {'embed_dim': 8, 'feat_name': 'C3', 'feat_num': 5521},\n",
       " {'embed_dim': 8, 'feat_name': 'C4', 'feat_num': 4033},\n",
       " {'embed_dim': 8, 'feat_name': 'C5', 'feat_num': 56},\n",
       " {'embed_dim': 8, 'feat_name': 'C6', 'feat_num': 8},\n",
       " {'embed_dim': 8, 'feat_name': 'C7', 'feat_num': 3184},\n",
       " {'embed_dim': 8, 'feat_name': 'C8', 'feat_num': 93},\n",
       " {'embed_dim': 8, 'feat_name': 'C9', 'feat_num': 3},\n",
       " {'embed_dim': 8, 'feat_name': 'C10', 'feat_num': 2986},\n",
       " {'embed_dim': 8, 'feat_name': 'C11', 'feat_num': 2084},\n",
       " {'embed_dim': 8, 'feat_name': 'C12', 'feat_num': 5284},\n",
       " {'embed_dim': 8, 'feat_name': 'C13', 'feat_num': 1725},\n",
       " {'embed_dim': 8, 'feat_name': 'C14', 'feat_num': 24},\n",
       " {'embed_dim': 8, 'feat_name': 'C15', 'feat_num': 2035},\n",
       " {'embed_dim': 8, 'feat_name': 'C16', 'feat_num': 4724},\n",
       " {'embed_dim': 8, 'feat_name': 'C17', 'feat_num': 9},\n",
       " {'embed_dim': 8, 'feat_name': 'C18', 'feat_num': 1149},\n",
       " {'embed_dim': 8, 'feat_name': 'C19', 'feat_num': 547},\n",
       " {'embed_dim': 8, 'feat_name': 'C20', 'feat_num': 4},\n",
       " {'embed_dim': 8, 'feat_name': 'C21', 'feat_num': 5037},\n",
       " {'embed_dim': 8, 'feat_name': 'C22', 'feat_num': 8},\n",
       " {'embed_dim': 8, 'feat_name': 'C23', 'feat_num': 12},\n",
       " {'embed_dim': 8, 'feat_name': 'C24', 'feat_num': 2525},\n",
       " {'embed_dim': 8, 'feat_name': 'C25', 'feat_num': 40},\n",
       " {'embed_dim': 8, 'feat_name': 'C26', 'feat_num': 1939},\n",
       " {'embed_dim': 8, 'feat_name': 'I1', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I2', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I3', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I4', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I5', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I6', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I7', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I8', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I9', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I10', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I11', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I12', 'feat_num': 100},\n",
       " {'embed_dim': 8, 'feat_name': 'I13', 'feat_num': 100}]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[   1,  293, 2491, ...,    0,    0,    1],\n",
       "        [   1,   88,    0, ...,    1,    0,    1],\n",
       "        [   1,   17, 5197, ...,    1,    0,    0],\n",
       "        ...,\n",
       "        [   1,  355, 4284, ...,    3,    0,    0],\n",
       "        [   1,  192,   56, ...,    1,    0,    0],\n",
       "        [  75,   18, 2613, ...,    3,    0,    0]], dtype=int32),\n",
       " array([0, 0, 0, ..., 0, 0, 0], dtype=int32))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 111,  105,  695, ...,    3,    0,    0],\n",
       "        [ 102,  337, 2613, ...,    0,    0,    1],\n",
       "        [  75,  301,  155, ...,    1,    0,    0],\n",
       "        ...,\n",
       "        [  75,   86,  507, ...,    1,    1,    1],\n",
       "        [   1,  347, 2205, ...,    2,    1,    1],\n",
       "        [ 102,  125,    5, ...,    1,    1,    0]], dtype=int32),\n",
       " array([1, 0, 1, ..., 0, 0, 0], dtype=int32))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-20 07:55:46\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "numpy  : 1.19.5\n",
      "IPython: 5.5.0\n",
      "pandas : 1.1.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
