{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "from transformers import GPT2Config, GPT2Model, GPT2Tokenizer, GPT2LMHeadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "out = 'gpt2-117M-bahasa-cased'\n",
    "os.makedirs(out, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('gpt2-117M-bahasa-cased/vocab.json',\n",
       " 'gpt2-117M-bahasa-cased/merges.txt',\n",
       " 'gpt2-117M-bahasa-cased/special_tokens_map.json',\n",
       " 'gpt2-117M-bahasa-cased/added_tokens.json')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = GPT2Tokenizer('117m-bahasa/bahasa-vocab.json', \n",
    "                            '117m-bahasa/bahasa-merges.txt', do_lower_case = False)\n",
    "tokenizer.save_pretrained('gpt2-117M-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading TF weight global_step with shape []\n",
      "Loading TF weight model/h0/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h0/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h0/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h0/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h0/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h0/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h0/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h0/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h0/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h0/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h0/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h0/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h0/ln_1/b with shape [768]\n",
      "Loading TF weight model/h0/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h0/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h0/ln_1/g with shape [768]\n",
      "Loading TF weight model/h0/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h0/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h0/ln_2/b with shape [768]\n",
      "Loading TF weight model/h0/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h0/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h0/ln_2/g with shape [768]\n",
      "Loading TF weight model/h0/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h0/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h0/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h0/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h0/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h0/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h0/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h0/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h0/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h0/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h0/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h0/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h0/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h0/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h1/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h1/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h1/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h1/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h1/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h1/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h1/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h1/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h1/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h1/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h1/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h1/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h1/ln_1/b with shape [768]\n",
      "Loading TF weight model/h1/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h1/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h1/ln_1/g with shape [768]\n",
      "Loading TF weight model/h1/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h1/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h1/ln_2/b with shape [768]\n",
      "Loading TF weight model/h1/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h1/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h1/ln_2/g with shape [768]\n",
      "Loading TF weight model/h1/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h1/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h1/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h1/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h1/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h1/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h1/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h1/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h1/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h1/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h1/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h1/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h1/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h1/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h10/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h10/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h10/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h10/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h10/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h10/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h10/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h10/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h10/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h10/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h10/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h10/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h10/ln_1/b with shape [768]\n",
      "Loading TF weight model/h10/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h10/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h10/ln_1/g with shape [768]\n",
      "Loading TF weight model/h10/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h10/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h10/ln_2/b with shape [768]\n",
      "Loading TF weight model/h10/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h10/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h10/ln_2/g with shape [768]\n",
      "Loading TF weight model/h10/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h10/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h10/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h10/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h10/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h10/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h10/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h10/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h10/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h10/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h10/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h10/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h10/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h10/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h11/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h11/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h11/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h11/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h11/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h11/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h11/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h11/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h11/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h11/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h11/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h11/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h11/ln_1/b with shape [768]\n",
      "Loading TF weight model/h11/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h11/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h11/ln_1/g with shape [768]\n",
      "Loading TF weight model/h11/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h11/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h11/ln_2/b with shape [768]\n",
      "Loading TF weight model/h11/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h11/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h11/ln_2/g with shape [768]\n",
      "Loading TF weight model/h11/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h11/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h11/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h11/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h11/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h11/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h11/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h11/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h11/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h11/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h11/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h11/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h11/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h11/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h2/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h2/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h2/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h2/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h2/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h2/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h2/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h2/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h2/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h2/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h2/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h2/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h2/ln_1/b with shape [768]\n",
      "Loading TF weight model/h2/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h2/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h2/ln_1/g with shape [768]\n",
      "Loading TF weight model/h2/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h2/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h2/ln_2/b with shape [768]\n",
      "Loading TF weight model/h2/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h2/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h2/ln_2/g with shape [768]\n",
      "Loading TF weight model/h2/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h2/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h2/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h2/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h2/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h2/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h2/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h2/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h2/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h2/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h2/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h2/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h2/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h2/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h3/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h3/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h3/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h3/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h3/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h3/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h3/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h3/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h3/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h3/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h3/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h3/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h3/ln_1/b with shape [768]\n",
      "Loading TF weight model/h3/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h3/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h3/ln_1/g with shape [768]\n",
      "Loading TF weight model/h3/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h3/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h3/ln_2/b with shape [768]\n",
      "Loading TF weight model/h3/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h3/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h3/ln_2/g with shape [768]\n",
      "Loading TF weight model/h3/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h3/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h3/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h3/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h3/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h3/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h3/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h3/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h3/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h3/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h3/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h3/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h3/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h3/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h4/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h4/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h4/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h4/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h4/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h4/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h4/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h4/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h4/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h4/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h4/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h4/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h4/ln_1/b with shape [768]\n",
      "Loading TF weight model/h4/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h4/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h4/ln_1/g with shape [768]\n",
      "Loading TF weight model/h4/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h4/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h4/ln_2/b with shape [768]\n",
      "Loading TF weight model/h4/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h4/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h4/ln_2/g with shape [768]\n",
      "Loading TF weight model/h4/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h4/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h4/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h4/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h4/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h4/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h4/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h4/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h4/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h4/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h4/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h4/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h4/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h4/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h5/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h5/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h5/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h5/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h5/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h5/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h5/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h5/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h5/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h5/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h5/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h5/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h5/ln_1/b with shape [768]\n",
      "Loading TF weight model/h5/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h5/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h5/ln_1/g with shape [768]\n",
      "Loading TF weight model/h5/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h5/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h5/ln_2/b with shape [768]\n",
      "Loading TF weight model/h5/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h5/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h5/ln_2/g with shape [768]\n",
      "Loading TF weight model/h5/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h5/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h5/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h5/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h5/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h5/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h5/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h5/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h5/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h5/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h5/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h5/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h5/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h5/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h6/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h6/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h6/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h6/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h6/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h6/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h6/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h6/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h6/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h6/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h6/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h6/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h6/ln_1/b with shape [768]\n",
      "Loading TF weight model/h6/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h6/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h6/ln_1/g with shape [768]\n",
      "Loading TF weight model/h6/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h6/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h6/ln_2/b with shape [768]\n",
      "Loading TF weight model/h6/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h6/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h6/ln_2/g with shape [768]\n",
      "Loading TF weight model/h6/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h6/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h6/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h6/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h6/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h6/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h6/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h6/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h6/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h6/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h6/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h6/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h6/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h6/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h7/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h7/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h7/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h7/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h7/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h7/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h7/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h7/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h7/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h7/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h7/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h7/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h7/ln_1/b with shape [768]\n",
      "Loading TF weight model/h7/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h7/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h7/ln_1/g with shape [768]\n",
      "Loading TF weight model/h7/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h7/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h7/ln_2/b with shape [768]\n",
      "Loading TF weight model/h7/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h7/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h7/ln_2/g with shape [768]\n",
      "Loading TF weight model/h7/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h7/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h7/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h7/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h7/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h7/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h7/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h7/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h7/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h7/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h7/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h7/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h7/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h7/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h8/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h8/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h8/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h8/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h8/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h8/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h8/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h8/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h8/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h8/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h8/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h8/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h8/ln_1/b with shape [768]\n",
      "Loading TF weight model/h8/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h8/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h8/ln_1/g with shape [768]\n",
      "Loading TF weight model/h8/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h8/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h8/ln_2/b with shape [768]\n",
      "Loading TF weight model/h8/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h8/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h8/ln_2/g with shape [768]\n",
      "Loading TF weight model/h8/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h8/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h8/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h8/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h8/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h8/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h8/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h8/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h8/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h8/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h8/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h8/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h8/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h8/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/h9/attn/c_attn/b with shape [2304]\n",
      "Loading TF weight model/h9/attn/c_attn/b/adam_m with shape [2304]\n",
      "Loading TF weight model/h9/attn/c_attn/b/adam_v with shape [2304]\n",
      "Loading TF weight model/h9/attn/c_attn/w with shape [1, 768, 2304]\n",
      "Loading TF weight model/h9/attn/c_attn/w/adam_m with shape [1, 768, 2304]\n",
      "Loading TF weight model/h9/attn/c_attn/w/adam_v with shape [1, 768, 2304]\n",
      "Loading TF weight model/h9/attn/c_proj/b with shape [768]\n",
      "Loading TF weight model/h9/attn/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h9/attn/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h9/attn/c_proj/w with shape [1, 768, 768]\n",
      "Loading TF weight model/h9/attn/c_proj/w/adam_m with shape [1, 768, 768]\n",
      "Loading TF weight model/h9/attn/c_proj/w/adam_v with shape [1, 768, 768]\n",
      "Loading TF weight model/h9/ln_1/b with shape [768]\n",
      "Loading TF weight model/h9/ln_1/b/adam_m with shape [768]\n",
      "Loading TF weight model/h9/ln_1/b/adam_v with shape [768]\n",
      "Loading TF weight model/h9/ln_1/g with shape [768]\n",
      "Loading TF weight model/h9/ln_1/g/adam_m with shape [768]\n",
      "Loading TF weight model/h9/ln_1/g/adam_v with shape [768]\n",
      "Loading TF weight model/h9/ln_2/b with shape [768]\n",
      "Loading TF weight model/h9/ln_2/b/adam_m with shape [768]\n",
      "Loading TF weight model/h9/ln_2/b/adam_v with shape [768]\n",
      "Loading TF weight model/h9/ln_2/g with shape [768]\n",
      "Loading TF weight model/h9/ln_2/g/adam_m with shape [768]\n",
      "Loading TF weight model/h9/ln_2/g/adam_v with shape [768]\n",
      "Loading TF weight model/h9/mlp/c_fc/b with shape [3072]\n",
      "Loading TF weight model/h9/mlp/c_fc/b/adam_m with shape [3072]\n",
      "Loading TF weight model/h9/mlp/c_fc/b/adam_v with shape [3072]\n",
      "Loading TF weight model/h9/mlp/c_fc/w with shape [1, 768, 3072]\n",
      "Loading TF weight model/h9/mlp/c_fc/w/adam_m with shape [1, 768, 3072]\n",
      "Loading TF weight model/h9/mlp/c_fc/w/adam_v with shape [1, 768, 3072]\n",
      "Loading TF weight model/h9/mlp/c_proj/b with shape [768]\n",
      "Loading TF weight model/h9/mlp/c_proj/b/adam_m with shape [768]\n",
      "Loading TF weight model/h9/mlp/c_proj/b/adam_v with shape [768]\n",
      "Loading TF weight model/h9/mlp/c_proj/w with shape [1, 3072, 768]\n",
      "Loading TF weight model/h9/mlp/c_proj/w/adam_m with shape [1, 3072, 768]\n",
      "Loading TF weight model/h9/mlp/c_proj/w/adam_v with shape [1, 3072, 768]\n",
      "Loading TF weight model/ln_f/b with shape [768]\n",
      "Loading TF weight model/ln_f/b/adam_m with shape [768]\n",
      "Loading TF weight model/ln_f/b/adam_v with shape [768]\n",
      "Loading TF weight model/ln_f/g with shape [768]\n",
      "Loading TF weight model/ln_f/g/adam_m with shape [768]\n",
      "Loading TF weight model/ln_f/g/adam_v with shape [768]\n",
      "Loading TF weight model/wpe with shape [1024, 768]\n",
      "Loading TF weight model/wpe/adam_m with shape [1024, 768]\n",
      "Loading TF weight model/wpe/adam_v with shape [1024, 768]\n",
      "Loading TF weight model/wte with shape [50257, 768]\n",
      "Loading TF weight model/wte/adam_m with shape [50257, 768]\n",
      "Loading TF weight model/wte/adam_v with shape [50257, 768]\n"
     ]
    }
   ],
   "source": [
    "tf_path = os.path.abspath('117m-bahasa-v3/model.ckpt-20000')\n",
    "init_vars = tf.train.list_variables(tf_path)\n",
    "names = []\n",
    "arrays = []\n",
    "for name, shape in init_vars:\n",
    "    print(\"Loading TF weight {} with shape {}\".format(name, shape))\n",
    "    array = tf.train.load_variable(tf_path, name)\n",
    "    names.append(name)\n",
    "    arrays.append(array.squeeze())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = GPT2Config.from_json_file('small-hparams.json')\n",
    "model = GPT2Model(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize PyTorch weight ['h0', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h0', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h0', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h0', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h0', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h0', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h0', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h0', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h0', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h0', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h0', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h0', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h1', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h1', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h1', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h1', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h1', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h1', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h1', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h1', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h1', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h1', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h1', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h1', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h10', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h10', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h10', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h10', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h10', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h10', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h10', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h10', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h10', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h10', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h10', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h10', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h11', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h11', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h11', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h11', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h11', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h11', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h11', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h11', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h11', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h11', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h11', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h11', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h2', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h2', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h2', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h2', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h2', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h2', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h2', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h2', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h2', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h2', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h2', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h2', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h3', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h3', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h3', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h3', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h3', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h3', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h3', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h3', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h3', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h3', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h3', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h3', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h4', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h4', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h4', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h4', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h4', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h4', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h4', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h4', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h4', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h4', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h4', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h4', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h5', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h5', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h5', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h5', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h5', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h5', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h5', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h5', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h5', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h5', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h5', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h5', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h6', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h6', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h6', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h6', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h6', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h6', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h6', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h6', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h6', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h6', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h6', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h6', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h7', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h7', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h7', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h7', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h7', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h7', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h7', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h7', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h7', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h7', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h7', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h7', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h8', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h8', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h8', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h8', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h8', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h8', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h8', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h8', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h8', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h8', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h8', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h8', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h9', 'attn', 'c_attn', 'b']\n",
      "Initialize PyTorch weight ['h9', 'attn', 'c_attn', 'w']\n",
      "Initialize PyTorch weight ['h9', 'attn', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h9', 'attn', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['h9', 'ln_1', 'b']\n",
      "Initialize PyTorch weight ['h9', 'ln_1', 'g']\n",
      "Initialize PyTorch weight ['h9', 'ln_2', 'b']\n",
      "Initialize PyTorch weight ['h9', 'ln_2', 'g']\n",
      "Initialize PyTorch weight ['h9', 'mlp', 'c_fc', 'b']\n",
      "Initialize PyTorch weight ['h9', 'mlp', 'c_fc', 'w']\n",
      "Initialize PyTorch weight ['h9', 'mlp', 'c_proj', 'b']\n",
      "Initialize PyTorch weight ['h9', 'mlp', 'c_proj', 'w']\n",
      "Initialize PyTorch weight ['ln_f', 'b']\n",
      "Initialize PyTorch weight ['ln_f', 'g']\n",
      "Initialize PyTorch weight ['wpe']\n",
      "Initialize PyTorch weight ['wte']\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "for name, array in zip(names, arrays):\n",
    "    name = name[6:]  # skip \"model/\"\n",
    "    name = name.split(\"/\")\n",
    "    pointer = model\n",
    "    passed = True\n",
    "    for m_name in name:\n",
    "        \n",
    "        if m_name in ['_step', 'adam_m', 'adam_v']:\n",
    "            passed = False\n",
    "            continue\n",
    "        if re.fullmatch(r\"[A-Za-z]+\\d+\", m_name):\n",
    "            scope_names = re.split(r\"(\\d+)\", m_name)\n",
    "        else:\n",
    "            scope_names = [m_name]\n",
    "        \n",
    "        if scope_names[0] == \"w\" or scope_names[0] == \"g\":\n",
    "            pointer = getattr(pointer, \"weight\")\n",
    "        elif scope_names[0] == \"b\":\n",
    "            pointer = getattr(pointer, \"bias\")\n",
    "        elif scope_names[0] == \"wpe\" or scope_names[0] == \"wte\":\n",
    "            pointer = getattr(pointer, scope_names[0])\n",
    "            pointer = getattr(pointer, \"weight\")\n",
    "        else:\n",
    "            pointer = getattr(pointer, scope_names[0])\n",
    "        if len(scope_names) >= 2:\n",
    "            num = int(scope_names[1])\n",
    "            pointer = pointer[num]\n",
    "    if passed:\n",
    "        try:\n",
    "            assert pointer.shape == array.shape\n",
    "        except AssertionError as e:\n",
    "            e.args += (pointer.shape, array.shape)\n",
    "            raise\n",
    "        print(\"Initialize PyTorch weight {}\".format(name))\n",
    "        pointer.data = torch.from_numpy(array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('config.json', 'pytorch_model.bin')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import CONFIG_NAME, WEIGHTS_NAME\n",
    "CONFIG_NAME, WEIGHTS_NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pytorch_dump_folder_path = out\n",
    "pytorch_weights_dump_path = pytorch_dump_folder_path + \"/\" + WEIGHTS_NAME\n",
    "pytorch_config_dump_path = pytorch_dump_folder_path + \"/\" + CONFIG_NAME\n",
    "\n",
    "torch.save(model.state_dict(), pytorch_weights_dump_path)\n",
    "\n",
    "with open(pytorch_config_dump_path, \"w\", encoding=\"utf-8\") as f:\n",
    "    f.write(config.to_json_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = GPT2Tokenizer.from_pretrained('./gpt2-117M-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = GPT2Config.from_json_file('small-hparams.json')\n",
    "model = GPT2Model.from_pretrained('./gpt2-117M-bahasa-cased/pytorch_model.bin', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained('gpt2-117M-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GPT2LMHeadModel.from_pretrained('./gpt2-117M-bahasa-cased', pad_token_id=tokenizer.eos_token_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2809,   266,  1681, 12167,    15,  1673,   656,   476,   558, 48854,\n",
       "          1470,   826]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('penat bak hang, macam ni aku takmau kerja dah', return_tensors='pt')\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output:\n",
      "----------------------------------------------------------------------------------------------------\n",
      "0: penat bak hang, macam ni aku takmau kerja dah jadi aku pernah beritahu orang.\n",
      "Ini bukan aku rasa cam nak ajak teman kan ni.\n",
      "Tengok ni aku dah ada adik-adik & anak yang tinggal dan kerja2 yang kat sekolah.\n",
      "1: penat bak hang, macam ni aku takmau kerja dah.\n",
      "Takleh takleh nak ambik air.\n",
      "Tgk jugak aku kat rumah ni.\n",
      "Pastu aku nak bagi aku.\n",
      "So aku dah takde masalah pulak.\n",
      "Balik aku pun\n",
      "2: penat bak hang, macam ni aku takmau kerja dah macam tu.\n",
      "Tapi semua tu aku ingat cakap, ada cara hidup ni yang kita kena bayar.. pastu kita tak mampu bayar.. kan!!\n",
      "Takpelah, aku nak cakap, masa yang\n"
     ]
    }
   ],
   "source": [
    "sample_outputs = model.generate(\n",
    "    input_ids,\n",
    "    do_sample=True, \n",
    "    max_length=50, \n",
    "    top_k=50, \n",
    "    top_p=0.95, \n",
    "    num_return_sequences=3\n",
    ")\n",
    "\n",
    "print(\"Output:\\n\" + 100 * '-')\n",
    "for i, sample_output in enumerate(sample_outputs):\n",
    "    print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !transformers-cli upload ./gpt2-117M-bahasa-cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a91615693274cbd990a79e13a0f984c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=874899.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc29de6ae55f4b4984a96fbbae18c3ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=432604.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51e1468b5c6d479881d939886c192660",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=90.0, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4abf05edfa84e258a2f9520e6644017",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = GPT2Tokenizer.from_pretrained('huseinzol05/gpt2-117M-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f384cbf135f4a69a64740c24a4016ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1346.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "551528e99cf24032acd328c6fb11c38e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=548117747.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = GPT2LMHeadModel.from_pretrained('huseinzol05/gpt2-117M-bahasa-cased', pad_token_id=tokenizer.eos_token_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2809,   266,  1681, 12167,    15,  1673,   656,   476,   558, 48854,\n",
       "          1470,   826]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('penat bak hang, macam ni aku takmau kerja dah', return_tensors='pt')\n",
    "input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_outputs = model.generate(\n",
    "    input_ids,\n",
    "    do_sample=True, \n",
    "    max_length=50, \n",
    "    top_k=50, \n",
    "    top_p=0.95, \n",
    "    num_return_sequences=3\n",
    ")\n",
    "\n",
    "print(\"Output:\\n\" + 100 * '-')\n",
    "for i, sample_output in enumerate(sample_outputs):\n",
    "    print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
