{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1,2,3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from gpt_2.src import model, encoder\n",
    "\n",
    "with open('bahasa-vocab.json', 'r') as f:\n",
    "    en = json.load(f)\n",
    "with open('bahasa-merges.txt', 'r', encoding=\"utf-8\") as f:\n",
    "    bpe_data = f.read()\n",
    "    \n",
    "bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\\n')[1:-1]]\n",
    "enc_malay = encoder.Encoder(\n",
    "    encoder=en,\n",
    "    bpe_merges=bpe_merges,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gpt_2_simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../bert/dumping-parliament.txt',\n",
       " '../bert/dumping-iium.txt',\n",
       " '../bert/dumping-wiki.txt',\n",
       " '../bert/dumping-news.txt',\n",
       " '../bert/dumping-watpadd.txt',\n",
       " '../bert/dumping-pdf.txt']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "files = glob('../bert/dumping-*.txt')\n",
    "files = [i for i in files if 'twitter' not in i and 'instagram' not in i and 'combined' not in i]\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../bert/dumping-parliament.txt',\n",
       " '../bert/dumping-iium.txt',\n",
       " '../bert/dumping-wiki.txt',\n",
       " '../bert/dumping-news.txt',\n",
       " '../bert/dumping-watpadd.txt',\n",
       " '../bert/dumping-pdf.txt',\n",
       " 'split.txtao',\n",
       " 'split.txtat',\n",
       " 'split.txtaj',\n",
       " 'split.txtad',\n",
       " 'split.txtaz',\n",
       " 'split.txtap',\n",
       " 'split.txtbb',\n",
       " 'split.txtbd',\n",
       " 'split.txtag',\n",
       " 'split.txtau',\n",
       " 'split.txtav',\n",
       " 'split.txtah',\n",
       " 'split.txtaq',\n",
       " 'split.txtab',\n",
       " 'split.txtai',\n",
       " 'split.txtba',\n",
       " 'split.txtax',\n",
       " 'split.txtar',\n",
       " 'split.txtac',\n",
       " 'split.txtam',\n",
       " 'split.txtas',\n",
       " 'split.txtae',\n",
       " 'split.txtaw',\n",
       " 'split.txtay',\n",
       " 'split.txtan',\n",
       " 'split.txtak',\n",
       " 'split.txtaa',\n",
       " 'split.txtal',\n",
       " 'split.txtaf',\n",
       " 'split.txtbc']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splitted = glob('split.*')\n",
    "files = files + splitted\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/device:GPU:0', '/device:GPU:1', '/device:GPU:2']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gpt_2_simple\n",
    "\n",
    "gpt_2_simple.get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gpt_2_simple.src import model, sample, encoder, memory_saving_gradients\n",
    "from gpt_2_simple.src.load_dataset import load_dataset, Sampler\n",
    "from gpt_2_simple.src.accumulate import AccumulatingOptimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_vocab': 50257, 'n_ctx': 1024, 'n_embd': 1024, 'n_head': 16, 'n_layer': 24}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hparams = model.default_hparams()\n",
    "with open('models/345M/hparams.json') as f:\n",
    "    hparams.override_from_dict(json.load(f))\n",
    "    \n",
    "hparams.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_memory_saving_gradients = True\n",
    "only_train_transformer_layers = True\n",
    "accumulate_gradients = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 36/36 [1:05:25<00:00, 109.05s/it]\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "combine = 50000\n",
    "\n",
    "token_chunks = []\n",
    "raw_text = ''\n",
    "paths = files\n",
    "\n",
    "for path in tqdm.tqdm(paths):\n",
    "    with open(path, 'r', encoding='utf8', errors='ignore') as fp:\n",
    "        raw_text += fp.read()\n",
    "    if len(raw_text) >= combine:\n",
    "        tokens = np.stack(enc_malay.encode(raw_text))\n",
    "        token_chunks.append(tokens)\n",
    "        raw_text = ''\n",
    "    else:\n",
    "        raw_text += '<|endoftext|>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(token_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('dataset.pkl', 'wb') as fopen:\n",
    "    pickle.dump(token_chunks, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([18111,    16, 18111,  4028,  9904,    16, 10427, 18784,   202,\n",
       "        1928,   323,  3071,  7485,  1378, 18600,  1378,  4322,  9198,\n",
       "         202,   202, 35594,   557,   725,   343,  7208,   805,  3679,\n",
       "        2815, 13254, 17746,   340, 12250,    54,  9229,   747,    64,\n",
       "         202, 20216,   551,    41, 17134,  6897,  6260,   847,  2113,\n",
       "         202, 46032,   699, 25654,  1942,   202, 26778,  2405,   202,\n",
       "         202,  1439, 12296,   557,  1046,   202, 38618,  9904,   281,\n",
       "        6683,  4638,   202, 10427, 10003,   202, 13120,   345, 24478,\n",
       "         499,  3467, 14036, 22143, 17294,  5229,   202,    37,   452,\n",
       "        5819, 37776,   340,   295,   747,   202, 23658,  7317, 42263,\n",
       "        3571,  4909, 17241,   311, 16059,   202,  8393,  1344,  8444,\n",
       "        3699])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_chunks[0][:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
