{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mkdir out-tiny\n",
    "# !gsutil cp gs://mesolitica-general/albert-tiny-actual/model.ckpt-70000.data-00000-of-00001 out-tiny\n",
    "# !gsutil cp gs://mesolitica-general/albert-tiny-actual/model.ckpt-70000.index out-tiny\n",
    "# !gsutil cp gs://mesolitica-general/albert-tiny-actual/model.ckpt-70000.meta out-tiny"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mkdir albert-base-2020-04-10\n",
    "# !cp sp10m.cased.v10.* albert-base-2020-04-10\n",
    "# !cp BASE_config.json albert-base-2020-04-10/config.json\n",
    "# !cp out/model.ckpt-400000* albert-base-2020-04-10\n",
    "# !tar cvzf albert-base-2020-04-10.tar.gz albert-base-2020-04-10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import modeling\n",
    "import optimization\n",
    "import tokenization\n",
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install sentencepiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loading sentence piece model\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tokenization.FullTokenizer(\n",
    "      vocab_file='sp10m.cased.v10.vocab', do_lower_case=False,\n",
    "      spm_model_file='sp10m.cased.v10.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['▁Hu', 'se', 'in', '▁comel']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize('Husein comel')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<modeling.AlbertConfig at 0x7f4f2d765b38>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "albert_config = modeling.AlbertConfig.from_json_file('TINY_config.json')\n",
    "albert_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gather_indexes(sequence_tensor, positions):\n",
    "    \"\"\"Gathers the vectors at the specific positions over a minibatch.\"\"\"\n",
    "    sequence_shape = modeling.get_shape_list(sequence_tensor, expected_rank=3)\n",
    "    batch_size = sequence_shape[0]\n",
    "    seq_length = sequence_shape[1]\n",
    "    width = sequence_shape[2]\n",
    "\n",
    "    flat_offsets = tf.reshape(\n",
    "      tf.range(0, batch_size, dtype=tf.int32) * seq_length, [-1, 1])\n",
    "    flat_positions = tf.reshape(positions + flat_offsets, [-1])\n",
    "    flat_sequence_tensor = tf.reshape(sequence_tensor,\n",
    "                                    [batch_size * seq_length, width])\n",
    "    output_tensor = tf.gather(flat_sequence_tensor, flat_positions)\n",
    "    return output_tensor\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.segment_ids = tf.placeholder(tf.int32, [None, None])\n",
    "        self.input_masks = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        model = modeling.AlbertModel(\n",
    "            config=albert_config,\n",
    "            is_training=False,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.input_masks,\n",
    "            token_type_ids=self.segment_ids,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        input_tensor = model.get_sequence_output()\n",
    "        output_weights = model.get_embedding_table()\n",
    "        \n",
    "        with tf.variable_scope(\"cls/predictions\"):\n",
    "            with tf.variable_scope(\"transform\"):\n",
    "                input_tensor = tf.layers.dense(\n",
    "                              input_tensor,\n",
    "                              units=albert_config.embedding_size,\n",
    "                              activation=modeling.get_activation(albert_config.hidden_act),\n",
    "                              kernel_initializer=modeling.create_initializer(\n",
    "                                  albert_config.initializer_range))\n",
    "                input_tensor = modeling.layer_norm(input_tensor)\n",
    "            \n",
    "            output_bias = tf.get_variable(\n",
    "                \"output_bias\",\n",
    "                shape=[albert_config.vocab_size],\n",
    "                initializer=tf.zeros_initializer())\n",
    "            logits = tf.matmul(input_tensor, output_weights, transpose_b=True)\n",
    "            logits = tf.nn.bias_add(logits, output_bias)\n",
    "            log_probs = tf.nn.log_softmax(logits, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/notebook/albert/modeling.py:256: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/ubuntu/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from out-tiny/model.ckpt-70000\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "cls = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'cls')\n",
    "saver = tf.train.Saver(var_list = var_lists + cls)\n",
    "saver.restore(sess, 'out-tiny/model.ckpt-70000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'albert-tiny/model.ckpt'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'albert-tiny/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !cp sp10m.cased.v10.* albert-base\n",
    "# !cp BASE_config.json albert-base/config.json\n",
    "# !tar cvzf albert-base.tar.gz albert-base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "out = 'albert-tiny-bahasa-cased'\n",
    "os.makedirs(out, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AlbertTokenizer, AlbertModel, AlbertConfig, AutoTokenizer, AutoModelWithLMHead, pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('albert-tiny-bahasa-cased/spiece.model',\n",
       " 'albert-tiny-bahasa-cased/special_tokens_map.json',\n",
       " 'albert-tiny-bahasa-cased/added_tokens.json')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AlbertTokenizer('sp10m.cased.v10.model', do_lower_case = False)\n",
    "tokenizer.save_pretrained('albert-tiny-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import logging\n",
    "from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_albert\n",
    "\n",
    "\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "\n",
    "def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_file, pytorch_dump_path):\n",
    "    # Initialise PyTorch model\n",
    "    config = AlbertConfig.from_json_file(albert_config_file)\n",
    "    print(\"Building PyTorch model from configuration: {}\".format(str(config)))\n",
    "    model = AlbertForMaskedLM(config)\n",
    "\n",
    "    # Load weights from tf checkpoint\n",
    "    load_tf_weights_in_albert(model, config, tf_checkpoint_path)\n",
    "\n",
    "    # Save pytorch-model\n",
    "    print(\"Save PyTorch model to {}\".format(pytorch_dump_path))\n",
    "    torch.save(model.state_dict(), pytorch_dump_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building PyTorch model from configuration: AlbertConfig {\n",
      "  \"_num_labels\": 2,\n",
      "  \"architectures\": null,\n",
      "  \"attention_probs_dropout_prob\": 0,\n",
      "  \"bos_token_id\": 2,\n",
      "  \"classifier_dropout_prob\": 0.1,\n",
      "  \"decoder_start_token_id\": null,\n",
      "  \"do_sample\": false,\n",
      "  \"down_scale_factor\": 1,\n",
      "  \"early_stopping\": false,\n",
      "  \"embedding_size\": 128,\n",
      "  \"eos_token_id\": 3,\n",
      "  \"finetuning_task\": null,\n",
      "  \"gap_size\": 0,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0,\n",
      "  \"hidden_size\": 336,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"inner_group_num\": 1,\n",
      "  \"intermediate_size\": 1344,\n",
      "  \"is_decoder\": false,\n",
      "  \"is_encoder_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"min_length\": 0,\n",
      "  \"model_type\": \"albert\",\n",
      "  \"net_structure_type\": 0,\n",
      "  \"no_repeat_ngram_size\": 0,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_groups\": 1,\n",
      "  \"num_hidden_layers\": 4,\n",
      "  \"num_memory_blocks\": 0,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"prefix\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"task_specific_params\": null,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 32000\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.modeling_albert:Converting TensorFlow checkpoint from /home/ubuntu/notebook/albert/albert-tiny/model.ckpt\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/embeddings/LayerNorm/beta with shape [128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/embeddings/LayerNorm/gamma with shape [128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/embeddings/position_embeddings with shape [512, 128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/embeddings/token_type_embeddings with shape [2, 128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/embeddings/word_embeddings with shape [32000, 128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/embedding_hidden_mapping_in/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/embedding_hidden_mapping_in/kernel with shape [128, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/LayerNorm/beta with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/LayerNorm/gamma with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/LayerNorm_1/beta with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/LayerNorm_1/gamma with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/output/dense/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/output/dense/kernel with shape [336, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/self/key/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/self/key/kernel with shape [336, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/self/query/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/self/query/kernel with shape [336, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/self/value/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/attention_1/self/value/kernel with shape [336, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/dense/bias with shape [1344]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/dense/kernel with shape [336, 1344]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/output/dense/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/output/dense/kernel with shape [1344, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/pooler/dense/bias with shape [336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight bert/pooler/dense/kernel with shape [336, 336]\n",
      "INFO:transformers.modeling_albert:Loading TF weight cls/predictions/output_bias with shape [32000]\n",
      "INFO:transformers.modeling_albert:Loading TF weight cls/predictions/transform/LayerNorm/beta with shape [128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight cls/predictions/transform/LayerNorm/gamma with shape [128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight cls/predictions/transform/dense/bias with shape [128]\n",
      "INFO:transformers.modeling_albert:Loading TF weight cls/predictions/transform/dense/kernel with shape [336, 128]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert/embeddings/LayerNorm/beta\n",
      "bert/embeddings/LayerNorm/gamma\n",
      "bert/embeddings/position_embeddings\n",
      "bert/embeddings/token_type_embeddings\n",
      "bert/embeddings/word_embeddings\n",
      "bert/encoder/embedding_hidden_mapping_in/bias\n",
      "bert/encoder/embedding_hidden_mapping_in/kernel\n",
      "bert/encoder/transformer/group_0/inner_group_0/LayerNorm/beta\n",
      "bert/encoder/transformer/group_0/inner_group_0/LayerNorm/gamma\n",
      "bert/encoder/transformer/group_0/inner_group_0/LayerNorm_1/beta\n",
      "bert/encoder/transformer/group_0/inner_group_0/LayerNorm_1/gamma\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/output/dense/bias\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/output/dense/kernel\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/self/key/bias\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/self/key/kernel\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/self/query/bias\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/self/query/kernel\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/self/value/bias\n",
      "bert/encoder/transformer/group_0/inner_group_0/attention_1/self/value/kernel\n",
      "bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/dense/bias\n",
      "bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/dense/kernel\n",
      "bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/output/dense/bias\n",
      "bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/output/dense/kernel\n",
      "bert/pooler/dense/bias\n",
      "bert/pooler/dense/kernel\n",
      "cls/predictions/output_bias\n",
      "cls/predictions/transform/LayerNorm/beta\n",
      "cls/predictions/transform/LayerNorm/gamma\n",
      "cls/predictions/transform/dense/bias\n",
      "cls/predictions/transform/dense/kernel\n",
      "Initialize PyTorch weight ['albert', 'embeddings', 'LayerNorm', 'beta'] from bert/embeddings/LayerNorm/beta\n",
      "Initialize PyTorch weight ['albert', 'embeddings', 'LayerNorm', 'gamma'] from bert/embeddings/LayerNorm/gamma\n",
      "Initialize PyTorch weight ['albert', 'embeddings', 'position_embeddings'] from bert/embeddings/position_embeddings\n",
      "Initialize PyTorch weight ['albert', 'embeddings', 'token_type_embeddings'] from bert/embeddings/token_type_embeddings\n",
      "Initialize PyTorch weight ['albert', 'embeddings', 'word_embeddings'] from bert/embeddings/word_embeddings\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'embedding_hidden_mapping_in', 'bias'] from bert/encoder/embedding_hidden_mapping_in/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'embedding_hidden_mapping_in', 'kernel'] from bert/encoder/embedding_hidden_mapping_in/kernel\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'LayerNorm', 'beta'] from bert/encoder/transformer/group_0/inner_group_0/LayerNorm/beta\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'LayerNorm', 'gamma'] from bert/encoder/transformer/group_0/inner_group_0/LayerNorm/gamma\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'full_layer_layer_norm', 'beta'] from bert/encoder/transformer/group_0/inner_group_0/LayerNorm_1/beta\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'full_layer_layer_norm', 'gamma'] from bert/encoder/transformer/group_0/inner_group_0/LayerNorm_1/gamma\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'dense', 'bias'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/output/dense/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'dense', 'kernel'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/output/dense/kernel\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'key', 'bias'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/self/key/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'key', 'kernel'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/self/key/kernel\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'query', 'bias'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/self/query/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'query', 'kernel'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/self/query/kernel\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'value', 'bias'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/self/value/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'attention', 'value', 'kernel'] from bert/encoder/transformer/group_0/inner_group_0/attention_1/self/value/kernel\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'ffn', 'bias'] from bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/dense/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'ffn', 'kernel'] from bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/dense/kernel\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'ffn_output', 'bias'] from bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/output/dense/bias\n",
      "Initialize PyTorch weight ['albert', 'encoder', 'albert_layer_groups', '0', 'albert_layers', '0', 'ffn_output', 'kernel'] from bert/encoder/transformer/group_0/inner_group_0/ffn_1/intermediate/output/dense/kernel\n",
      "Initialize PyTorch weight ['albert', 'pooler', 'bias'] from bert/pooler/dense/bias\n",
      "Initialize PyTorch weight ['albert', 'pooler', 'kernel'] from bert/pooler/dense/kernel\n",
      "Initialize PyTorch weight ['predictions', 'output_bias'] from cls/predictions/output_bias\n",
      "Initialize PyTorch weight ['predictions', 'LayerNorm', 'beta'] from cls/predictions/transform/LayerNorm/beta\n",
      "Initialize PyTorch weight ['predictions', 'LayerNorm', 'gamma'] from cls/predictions/transform/LayerNorm/gamma\n",
      "Initialize PyTorch weight ['predictions', 'dense', 'bias'] from cls/predictions/transform/dense/bias\n",
      "Initialize PyTorch weight ['predictions', 'dense', 'kernel'] from cls/predictions/transform/dense/kernel\n",
      "Save PyTorch model to albert-tiny-bahasa-cased/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "convert_tf_checkpoint_to_pytorch('albert-tiny/model.ckpt', \n",
    "                                 'TINY_config.json', \n",
    "                                 'albert-tiny-bahasa-cased/pytorch_model.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.tokenization_utils:Model name './albert-tiny-bahasa-cased' not found in model shortcut name list (albert-base-v1, albert-large-v1, albert-xlarge-v1, albert-xxlarge-v1, albert-base-v2, albert-large-v2, albert-xlarge-v2, albert-xxlarge-v2). Assuming './albert-tiny-bahasa-cased' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "INFO:transformers.tokenization_utils:Didn't find file ./albert-tiny-bahasa-cased/added_tokens.json. We won't load it.\n",
      "INFO:transformers.tokenization_utils:loading file ./albert-tiny-bahasa-cased/spiece.model\n",
      "INFO:transformers.tokenization_utils:loading file None\n",
      "INFO:transformers.tokenization_utils:loading file ./albert-tiny-bahasa-cased/special_tokens_map.json\n",
      "INFO:transformers.tokenization_utils:loading file ./albert-tiny-bahasa-cased/tokenizer_config.json\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AlbertTokenizer.from_pretrained('./albert-tiny-bahasa-cased', do_lower_case = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = AlbertConfig('TINY_config.json')\n",
    "config.vocab_size = 32000\n",
    "config.intermediate_size = 1344\n",
    "config.hidden_size = 336\n",
    "config.num_attention_heads = 12\n",
    "config.num_hidden_groups = 1\n",
    "config.num_hidden_layers = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.modeling_utils:loading weights file ./albert-tiny-bahasa-cased/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelWithLMHead.from_pretrained('./albert-tiny-bahasa-cased/pytorch_model.bin', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "fill_mask = pipeline('fill-mask', model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'sequence': '[CLS] tolonglah gov buat something, kami dah kahwin[SEP]',\n",
       "  'score': 0.020490359514951706,\n",
       "  'token': 1326},\n",
       " {'sequence': '[CLS] tolonglah gov buat something, kami dah?[SEP]',\n",
       "  'score': 0.011642860248684883,\n",
       "  'token': 28},\n",
       " {'sequence': '[CLS] tolonglah gov buat something, kami dah tahu[SEP]',\n",
       "  'score': 0.007895083166658878,\n",
       "  'token': 178},\n",
       " {'sequence': '[CLS] tolonglah gov buat something, kami dah lama[SEP]',\n",
       "  'score': 0.00789356604218483,\n",
       "  'token': 222},\n",
       " {'sequence': '[CLS] tolonglah gov buat something, kami dah ni[SEP]',\n",
       "  'score': 0.007744117174297571,\n",
       "  'token': 34}]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask('tolonglah gov buat something, kami dah [MASK]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.configuration_utils:Configuration saved in albert-tiny-bahasa-cased/config.json\n",
      "INFO:transformers.modeling_utils:Model weights saved in albert-tiny-bahasa-cased/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "model.save_pretrained('albert-tiny-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !transformers-cli upload ./albert-tiny-bahasa-cased"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:filelock:Lock 139976865798072 acquired on /home/ubuntu/.cache/torch/transformers/012922c7e0fdfe2ddc58e35274debe862ce8a2dc627e3f5cb784f983ae8f3be6.de6d44e60aa3f72beeb9726ba6ff879337721eb27c49f517dc145e348b7a34d4.lock\n",
      "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/pytorch_model.bin not found in cache or force_download set to True, downloading to /home/ubuntu/.cache/torch/transformers/tmpqp10_0_j\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14377e72a51b46598d3aa4f1bddb9a92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=23019367.0, style=ProgressStyle(descrip…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/pytorch_model.bin in cache at /home/ubuntu/.cache/torch/transformers/012922c7e0fdfe2ddc58e35274debe862ce8a2dc627e3f5cb784f983ae8f3be6.de6d44e60aa3f72beeb9726ba6ff879337721eb27c49f517dc145e348b7a34d4\n",
      "INFO:transformers.file_utils:creating metadata file for /home/ubuntu/.cache/torch/transformers/012922c7e0fdfe2ddc58e35274debe862ce8a2dc627e3f5cb784f983ae8f3be6.de6d44e60aa3f72beeb9726ba6ff879337721eb27c49f517dc145e348b7a34d4\n",
      "INFO:filelock:Lock 139976865798072 released on /home/ubuntu/.cache/torch/transformers/012922c7e0fdfe2ddc58e35274debe862ce8a2dc627e3f5cb784f983ae8f3be6.de6d44e60aa3f72beeb9726ba6ff879337721eb27c49f517dc145e348b7a34d4.lock\n",
      "INFO:transformers.modeling_utils:loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/pytorch_model.bin from cache at /home/ubuntu/.cache/torch/transformers/012922c7e0fdfe2ddc58e35274debe862ce8a2dc627e3f5cb784f983ae8f3be6.de6d44e60aa3f72beeb9726ba6ff879337721eb27c49f517dc145e348b7a34d4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelWithLMHead.from_pretrained('huseinzol05/albert-tiny-bahasa-cased', config = config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.tokenization_utils:Model name 'huseinzol05/albert-tiny-bahasa-cased' not found in model shortcut name list (albert-base-v1, albert-large-v1, albert-xlarge-v1, albert-xxlarge-v1, albert-base-v2, albert-large-v2, albert-xlarge-v2, albert-xxlarge-v2). Assuming 'huseinzol05/albert-tiny-bahasa-cased' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "INFO:filelock:Lock 139976863721400 acquired on /home/ubuntu/.cache/torch/transformers/9af39e08496cfbc2beca52ea108da93f4e0faf2165f204dfe41625af07cc265c.62912bc1f6182c2bdac801dba22c51182bb7bdbc199b220c540bbb4dada8ed34.lock\n",
      "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/spiece.model not found in cache or force_download set to True, downloading to /home/ubuntu/.cache/torch/transformers/tmpu9eoku8_\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44e4e47a91394b50bd56c9b58e358a4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=778641.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/spiece.model in cache at /home/ubuntu/.cache/torch/transformers/9af39e08496cfbc2beca52ea108da93f4e0faf2165f204dfe41625af07cc265c.62912bc1f6182c2bdac801dba22c51182bb7bdbc199b220c540bbb4dada8ed34\n",
      "INFO:transformers.file_utils:creating metadata file for /home/ubuntu/.cache/torch/transformers/9af39e08496cfbc2beca52ea108da93f4e0faf2165f204dfe41625af07cc265c.62912bc1f6182c2bdac801dba22c51182bb7bdbc199b220c540bbb4dada8ed34\n",
      "INFO:filelock:Lock 139976863721400 released on /home/ubuntu/.cache/torch/transformers/9af39e08496cfbc2beca52ea108da93f4e0faf2165f204dfe41625af07cc265c.62912bc1f6182c2bdac801dba22c51182bb7bdbc199b220c540bbb4dada8ed34.lock\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:filelock:Lock 139978021282872 acquired on /home/ubuntu/.cache/torch/transformers/aa2391aa261ccc65ae4b6269f8e7db027364d5f1b76fbf52ff4b41561d644810.4f0d42b1849e2d6fd72c735fba48dff0d2f0a55f5d1961e79bcfce337d354167.lock\n",
      "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/special_tokens_map.json not found in cache or force_download set to True, downloading to /home/ubuntu/.cache/torch/transformers/tmp4xauqpf0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7dffa9bc9144eff964318c5c84cbc3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=156.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/special_tokens_map.json in cache at /home/ubuntu/.cache/torch/transformers/aa2391aa261ccc65ae4b6269f8e7db027364d5f1b76fbf52ff4b41561d644810.4f0d42b1849e2d6fd72c735fba48dff0d2f0a55f5d1961e79bcfce337d354167\n",
      "INFO:transformers.file_utils:creating metadata file for /home/ubuntu/.cache/torch/transformers/aa2391aa261ccc65ae4b6269f8e7db027364d5f1b76fbf52ff4b41561d644810.4f0d42b1849e2d6fd72c735fba48dff0d2f0a55f5d1961e79bcfce337d354167\n",
      "INFO:filelock:Lock 139978021282872 released on /home/ubuntu/.cache/torch/transformers/aa2391aa261ccc65ae4b6269f8e7db027364d5f1b76fbf52ff4b41561d644810.4f0d42b1849e2d6fd72c735fba48dff0d2f0a55f5d1961e79bcfce337d354167.lock\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:filelock:Lock 139976865793584 acquired on /home/ubuntu/.cache/torch/transformers/af67b61a872b064e1b4e3f539ecb7aa59f4700db79093e8c74f3809e5d7e946b.3889713104075cfee9e96090bcdd0dc753733b3db9da20d1dd8b2cd1030536a2.lock\n",
      "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/tokenizer_config.json not found in cache or force_download set to True, downloading to /home/ubuntu/.cache/torch/transformers/tmp5_srz_sh\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8c04bcdac234e1abdde16bfe2356481",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/tokenizer_config.json in cache at /home/ubuntu/.cache/torch/transformers/af67b61a872b064e1b4e3f539ecb7aa59f4700db79093e8c74f3809e5d7e946b.3889713104075cfee9e96090bcdd0dc753733b3db9da20d1dd8b2cd1030536a2\n",
      "INFO:transformers.file_utils:creating metadata file for /home/ubuntu/.cache/torch/transformers/af67b61a872b064e1b4e3f539ecb7aa59f4700db79093e8c74f3809e5d7e946b.3889713104075cfee9e96090bcdd0dc753733b3db9da20d1dd8b2cd1030536a2\n",
      "INFO:filelock:Lock 139976865793584 released on /home/ubuntu/.cache/torch/transformers/af67b61a872b064e1b4e3f539ecb7aa59f4700db79093e8c74f3809e5d7e946b.3889713104075cfee9e96090bcdd0dc753733b3db9da20d1dd8b2cd1030536a2.lock\n",
      "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/spiece.model from cache at /home/ubuntu/.cache/torch/transformers/9af39e08496cfbc2beca52ea108da93f4e0faf2165f204dfe41625af07cc265c.62912bc1f6182c2bdac801dba22c51182bb7bdbc199b220c540bbb4dada8ed34\n",
      "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/added_tokens.json from cache at None\n",
      "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/special_tokens_map.json from cache at /home/ubuntu/.cache/torch/transformers/aa2391aa261ccc65ae4b6269f8e7db027364d5f1b76fbf52ff4b41561d644810.4f0d42b1849e2d6fd72c735fba48dff0d2f0a55f5d1961e79bcfce337d354167\n",
      "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/huseinzol05/albert-tiny-bahasa-cased/tokenizer_config.json from cache at /home/ubuntu/.cache/torch/transformers/af67b61a872b064e1b4e3f539ecb7aa59f4700db79093e8c74f3809e5d7e946b.3889713104075cfee9e96090bcdd0dc753733b3db9da20d1dd8b2cd1030536a2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AlbertTokenizer.from_pretrained('huseinzol05/albert-tiny-bahasa-cased', do_lower_case = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'sequence': '[CLS] makan ayam dengan ayam[SEP]',\n",
       "  'score': 0.05121927708387375,\n",
       "  'token': 629},\n",
       " {'sequence': '[CLS] makan ayam dengan sayur[SEP]',\n",
       "  'score': 0.04497420787811279,\n",
       "  'token': 1639},\n",
       " {'sequence': '[CLS] makan ayam dengan nasi[SEP]',\n",
       "  'score': 0.039827536791563034,\n",
       "  'token': 453},\n",
       " {'sequence': '[CLS] makan ayam dengan rendang[SEP]',\n",
       "  'score': 0.032997727394104004,\n",
       "  'token': 2451},\n",
       " {'sequence': '[CLS] makan ayam dengan makan[SEP]',\n",
       "  'score': 0.031354598701000214,\n",
       "  'token': 129}]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill_mask = pipeline('fill-mask', model=model, tokenizer=tokenizer)\n",
    "fill_mask('makan ayam dengan [MASK]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
