{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from unidecode import unidecode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import six\n",
    "from functools import partial\n",
    "\n",
    "SPIECE_UNDERLINE = '▁'\n",
    "\n",
    "def preprocess_text(inputs, lower=False, remove_space=True, keep_accents=False):\n",
    "  if remove_space:\n",
    "    outputs = ' '.join(inputs.strip().split())\n",
    "  else:\n",
    "    outputs = inputs\n",
    "  outputs = outputs.replace(\"``\", '\"').replace(\"''\", '\"')\n",
    "\n",
    "  if six.PY2 and isinstance(outputs, str):\n",
    "    outputs = outputs.decode('utf-8')\n",
    "\n",
    "  if not keep_accents:\n",
    "    outputs = unicodedata.normalize('NFKD', outputs)\n",
    "    outputs = ''.join([c for c in outputs if not unicodedata.combining(c)])\n",
    "  if lower:\n",
    "    outputs = outputs.lower()\n",
    "\n",
    "  return outputs\n",
    "\n",
    "\n",
    "def encode_pieces(sp_model, text, return_unicode=True, sample=False):\n",
    "  # return_unicode is used only for py2\n",
    "\n",
    "  # note(zhiliny): in some systems, sentencepiece only accepts str for py2\n",
    "  if six.PY2 and isinstance(text, unicode):\n",
    "    text = text.encode('utf-8')\n",
    "\n",
    "  if not sample:\n",
    "    pieces = sp_model.EncodeAsPieces(text)\n",
    "  else:\n",
    "    pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1)\n",
    "  new_pieces = []\n",
    "  for piece in pieces:\n",
    "    if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit():\n",
    "      cur_pieces = sp_model.EncodeAsPieces(\n",
    "          piece[:-1].replace(SPIECE_UNDERLINE, ''))\n",
    "      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:\n",
    "        if len(cur_pieces[0]) == 1:\n",
    "          cur_pieces = cur_pieces[1:]\n",
    "        else:\n",
    "          cur_pieces[0] = cur_pieces[0][1:]\n",
    "      cur_pieces.append(piece[-1])\n",
    "      new_pieces.extend(cur_pieces)\n",
    "    else:\n",
    "      new_pieces.append(piece)\n",
    "\n",
    "  # note(zhiliny): convert back to unicode for py2\n",
    "  if six.PY2 and return_unicode:\n",
    "    ret_pieces = []\n",
    "    for piece in new_pieces:\n",
    "      if isinstance(piece, str):\n",
    "        piece = piece.decode('utf-8')\n",
    "      ret_pieces.append(piece)\n",
    "    new_pieces = ret_pieces\n",
    "\n",
    "  return new_pieces\n",
    "\n",
    "\n",
    "def encode_ids(sp_model, text, sample=False):\n",
    "  pieces = encode_pieces(sp_model, text, return_unicode=False, sample=sample)\n",
    "  ids = [sp_model.PieceToId(piece) for piece in pieces]\n",
    "  return ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v):\n",
    "        self.vocab = v\n",
    "        pass\n",
    "    \n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(sp_model, string, return_unicode=False, sample=False)\n",
    "    \n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [sp_model.PieceToId(piece) for piece in tokens]\n",
    "    \n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [sp_model.IdToPiece(i) for i in ids]\n",
    "    \n",
    "tokenizer = Tokenizer(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['train_X', 'test_X', 'train_Y', 'test_Y'])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../session-entities.pkl', 'rb') as fopen:\n",
    "    data = pickle.load(fopen)\n",
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = data['train_X']\n",
    "test_X = data['test_X']\n",
    "train_Y = data['train_Y']\n",
    "test_Y = data['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['word2idx', 'idx2word', 'tag2idx', 'idx2tag', 'char2idx'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../dictionary-entities.json') as fopen:\n",
    "    dictionary = json.load(fopen)\n",
    "dictionary.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = dictionary['word2idx']\n",
    "idx2word = {int(k): v for k, v in dictionary['idx2word'].items()}\n",
    "tag2idx = dictionary['tag2idx']\n",
    "idx2tag = {int(k): v for k, v in dictionary['idx2tag'].items()}\n",
    "char2idx = dictionary['char2idx']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def XY(strings):\n",
    "    left_train, right_train = strings[0], strings[1]\n",
    "    X, Y, MASK = [], [], []\n",
    "    for i in tqdm(range(len(left_train))):\n",
    "        left = [idx2word[d] for d in left_train[i]]\n",
    "        right = [idx2tag[d] for d in right_train[i]]\n",
    "        bert_tokens = ['[CLS]']\n",
    "        y = ['PAD']\n",
    "        for no, orig_token in enumerate(left):\n",
    "            t = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(t)\n",
    "            if len(t):\n",
    "                y.append(right[no])\n",
    "            y.extend(['X'] * (len(t) - 1))\n",
    "        bert_tokens.append('[SEP]')\n",
    "        y.append('PAD')\n",
    "        x = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "        y = [tag2idx[i] for i in y]\n",
    "        input_mask = [1] * len(y)\n",
    "        if len(x) != len(y):\n",
    "            print(i)\n",
    "        X.append(x)\n",
    "        Y.append(y)\n",
    "        MASK.append(input_mask)\n",
    "    return X, Y, MASK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 36762/36762 [00:24<00:00, 1524.85it/s]\n",
      "100%|██████████| 36762/36762 [00:24<00:00, 1486.38it/s]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]0:00, 1487.83it/s]\n",
      "100%|██████████| 7/7 [00:00<00:00, 1665.35it/s].68it/s]\n",
      "100%|██████████| 36762/36762 [00:24<00:00, 1511.90it/s]\n",
      "100%|██████████| 36762/36762 [00:24<00:00, 1481.28it/s]\n",
      " 99%|█████████▉| 36369/36762 [00:24<00:00, 1486.88it/s]\n",
      "100%|██████████| 36762/36762 [00:24<00:00, 1501.21it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1466.84it/s]\n",
      "100%|██████████| 36762/36762 [00:24<00:00, 1478.04it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1452.14it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1421.30it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1463.04it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1434.37it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1449.55it/s]\n",
      "100%|██████████| 36762/36762 [00:25<00:00, 1418.51it/s]\n",
      "100%|██████████| 36762/36762 [00:26<00:00, 1404.07it/s]\n"
     ]
    }
   ],
   "source": [
    "train_X, train_Y, train_masks = cleaning.multiprocessing(train_X, train_Y, XY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 8313/9188 [00:06<00:00, 1487.27it/s]\n",
      "100%|██████████| 5/5 [00:00<00:00, 1537.39it/s]6it/s]\n",
      " 99%|█████████▉| 9091/9188 [00:06<00:00, 1575.31it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1501.07it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1485.41it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1450.28it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1456.25it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1465.78it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1493.39it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1435.83it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1455.88it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1445.92it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1409.87it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1386.31it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1404.18it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1378.58it/s]\n",
      "100%|██████████| 9188/9188 [00:06<00:00, 1351.35it/s]\n"
     ]
    }
   ],
   "source": [
    "test_X, test_Y, test_masks = cleaning.multiprocessing(test_X, test_Y, XY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def XY_extra(left_train, right_train):\n",
    "    X, Y, MASK = [], [], []\n",
    "    for i in tqdm(range(len(left_train))):\n",
    "        left = left_train[i]\n",
    "        right = right_train[i]\n",
    "        bert_tokens = ['[CLS]']\n",
    "        y = ['PAD']\n",
    "        for no, orig_token in enumerate(left):\n",
    "            t = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(t)\n",
    "            if len(t):\n",
    "                y.append(right[no])\n",
    "            y.extend(['X'] * (len(t) - 1))\n",
    "        bert_tokens.append('[SEP]')\n",
    "        y.append('PAD')\n",
    "        x = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "        y = [tag2idx[i] for i in y]\n",
    "        input_mask = [1] * len(y)\n",
    "        if len(x) != len(y):\n",
    "            print(i)\n",
    "        X.append(x)\n",
    "        Y.append(y)\n",
    "        MASK.append(input_mask)\n",
    "    return X, Y, MASK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['X', 'Y'])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../extra-entities.json') as fopen:\n",
    "    extra = json.load(fopen)\n",
    "    \n",
    "extra.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 236143/236143 [01:21<00:00, 2887.45it/s]\n"
     ]
    }
   ],
   "source": [
    "extra_X, extra_Y, extra_masks = XY_extra(extra['X'], extra['Y'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X.extend(extra_X)\n",
    "train_Y.extend(extra_Y)\n",
    "train_masks.extend(extra_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, train_Y, train_masks = shuffle(train_X, train_Y, train_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'bert-base-v3/model.ckpt'\n",
    "BERT_CONFIG = 'bert-base-v3/config.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "epoch = 5\n",
    "batch_size = 32\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(1000 / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)\n",
    "bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_initializer(initializer_range=0.02):\n",
    "    return tf.truncated_normal_initializer(stddev=initializer_range)\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 2e-5,\n",
    "        training = True\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.MASK = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.maxlen = tf.shape(self.X)[1]\n",
    "        self.lengths = tf.count_nonzero(self.X, 1)\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.MASK,\n",
    "            use_one_hot_embeddings=False)\n",
    "        output_layer = model.get_sequence_output()\n",
    "        output_layer = tf.layers.dense(\n",
    "            output_layer,\n",
    "            bert_config.hidden_size,\n",
    "            activation=tf.tanh,\n",
    "            kernel_initializer=create_initializer())\n",
    "        logits = tf.layers.dense(output_layer, dimension_output,\n",
    "                                         kernel_initializer=create_initializer())\n",
    "        y_t = self.Y\n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            logits, y_t, self.lengths\n",
    "        )\n",
    "        self.cost = tf.reduce_mean(-log_likelihood)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        self.tags_seq, tags_score = tf.contrib.crf.crf_decode(\n",
    "            logits, transition_params, self.lengths\n",
    "        )\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'logits')\n",
    "\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n",
    "        mask_label = tf.boolean_mask(y_t, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:99: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "INFO:tensorflow:Restoring parameters from bert-base-v3/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = len(tag2idx)\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['[CLS]', '[SEP]']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in rejected\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in rejected]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'KUALA LUMPUR: Sempena sambutan Aidilfitri minggu depan, Perdana Menteri Tun Dr Mahathir Mohamad dan Menteri Pengangkutan Anthony Loke Siew Fook menitipkan pesanan khas kepada orang ramai yang mahu pulang ke kampung halaman masing-masing. Dalam video pendek terbitan Jabatan Keselamatan Jalan Raya (JKJR) itu, Dr Mahathir menasihati mereka supaya berhenti berehat dan tidur sebentar  sekiranya mengantuk ketika memandu.'\n",
    "\n",
    "import re\n",
    "\n",
    "def entities_textcleaning(string, lowering = False):\n",
    "    \"\"\"\n",
    "    use by entities recognition, pos recognition and dependency parsing\n",
    "    \"\"\"\n",
    "    string = re.sub('[^A-Za-z0-9\\-\\/() ]+', ' ', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    original_string = string.split()\n",
    "    if lowering:\n",
    "        string = string.lower()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    bert_tokens = ['[CLS]']\n",
    "    for no, orig_token in enumerate(left):\n",
    "        t = tokenizer.tokenize(orig_token)\n",
    "        bert_tokens.extend(t)\n",
    "    bert_tokens.append(\"[SEP]\")\n",
    "    input_mask = [1] * len(bert_tokens)\n",
    "    return tokenizer.convert_tokens_to_ids(bert_tokens), bert_tokens, input_mask\n",
    "\n",
    "sequence = entities_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, input_mask = parse_X(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Kuala', 'PAD'),\n",
       " ('Lumpur', 'organization'),\n",
       " ('Sempena', 'time'),\n",
       " ('sambutan', 'organization'),\n",
       " ('Aidilfitri', 'time'),\n",
       " ('minggu', 'organization'),\n",
       " ('depan', 'time'),\n",
       " ('Perdana', 'X'),\n",
       " ('Menteri', 'OTHER'),\n",
       " ('Tun', 'quantity'),\n",
       " ('Dr', 'location'),\n",
       " ('Mahathir', 'time'),\n",
       " ('Mohamad', 'X'),\n",
       " ('dan', 'organization'),\n",
       " ('Menteri', 'time'),\n",
       " ('Pengangkutan', 'organization'),\n",
       " ('Anthony', 'time'),\n",
       " ('Loke', 'X'),\n",
       " ('Siew', 'organization'),\n",
       " ('Fook', 'time'),\n",
       " ('menitipkan', 'organization'),\n",
       " ('pesanan', 'time'),\n",
       " ('khas', 'organization'),\n",
       " ('kepada', 'time'),\n",
       " ('orang', 'X'),\n",
       " ('ramai', 'OTHER'),\n",
       " ('yang', 'X'),\n",
       " ('mahu', 'OTHER'),\n",
       " ('pulang', 'quantity'),\n",
       " ('ke', 'X'),\n",
       " ('kampung', 'OTHER'),\n",
       " ('halaman', 'quantity'),\n",
       " ('masing-masing', 'law'),\n",
       " ('Dalam', 'time'),\n",
       " ('video', 'organization'),\n",
       " ('pendek', 'time'),\n",
       " ('terbitan', 'organization'),\n",
       " ('Jabatan', 'PAD'),\n",
       " ('Keselamatan', 'organization'),\n",
       " ('Jalan', 'PAD'),\n",
       " ('Raya', 'organization'),\n",
       " ('(Jkjr)', 'PAD'),\n",
       " ('itu', 'time'),\n",
       " ('Dr', 'organization'),\n",
       " ('Mahathir', 'time'),\n",
       " ('menasihati', 'organization'),\n",
       " ('mereka', 'PAD'),\n",
       " ('supaya', 'PAD'),\n",
       " ('berhenti', 'organization'),\n",
       " ('berehat', 'time'),\n",
       " ('dan', 'organization'),\n",
       " ('tidur', 'time'),\n",
       " ('sebentar', 'organization'),\n",
       " ('sekiranya', 'location'),\n",
       " ('mengantuk', 'time'),\n",
       " ('ketika', 'X'),\n",
       " ('memandu', 'OTHER')]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted = sess.run(model.tags_seq,\n",
    "                feed_dict = {\n",
    "                    model.X: [parsed_sequence],\n",
    "                    model.MASK: [input_mask]\n",
    "                },\n",
    "        )[0]\n",
    "merged = merge_sentencepiece_tokens_tagging(bert_sequence, [idx2tag[d] for d in predicted])\n",
    "list(zip(merged[0], merged[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  11%|█         | 2790/25761 [18:02<2:18:23,  2.77it/s, accuracy=0.997, cost=0.652]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  38%|███▊      | 9763/25761 [1:01:30<1:43:22,  2.58it/s, accuracy=1, cost=0.0561]    IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  54%|█████▍    | 13983/25761 [1:29:02<1:10:51,  2.77it/s, accuracy=1, cost=0.0266]    IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  71%|███████   | 18194/25761 [1:57:53<45:24,  2.78it/s, accuracy=1, cost=0.0191]      IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 25761/25761 [2:45:13<00:00,  2.60it/s, accuracy=1, cost=0.0218]    \n",
      "test minibatch loop: 100%|██████████| 4595/4595 [11:42<00:00,  6.54it/s, accuracy=0.997, cost=0.424] \n",
      "train minibatch loop:   0%|          | 0/25761 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 10616.014313220978\n",
      "epoch: 0, training loss: 0.482649, training acc: 0.997700, valid loss: 4.194917, valid acc: 0.987690\n",
      "\n",
      "[('Kuala', 'location'), ('Lumpur', 'location'), ('Sempena', 'OTHER'), ('sambutan', 'OTHER'), ('Aidilfitri', 'event'), ('minggu', 'time'), ('depan', 'time'), ('Perdana', 'person'), ('Menteri', 'person'), ('Tun', 'person'), ('Dr', 'person'), ('Mahathir', 'person'), ('Mohamad', 'person'), ('dan', 'OTHER'), ('Menteri', 'person'), ('Pengangkutan', 'person'), ('Anthony', 'person'), ('Loke', 'person'), ('Siew', 'person'), ('Fook', 'person'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'location'), ('masing-masing', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'organization'), ('Keselamatan', 'organization'), ('Jalan', 'organization'), ('Raya', 'organization'), ('(Jkjr)', 'organization'), ('itu', 'OTHER'), ('Dr', 'person'), ('Mahathir', 'person'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 25761/25761 [2:41:03<00:00,  2.67it/s, accuracy=1, cost=0.00128]     \n",
      "test minibatch loop: 100%|██████████| 4595/4595 [11:40<00:00,  6.56it/s, accuracy=1, cost=0.0068]    \n",
      "train minibatch loop:   0%|          | 0/25761 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 10364.319827318192\n",
      "epoch: 1, training loss: 0.049287, training acc: 0.999759, valid loss: 3.895352, valid acc: 0.988612\n",
      "\n",
      "[('Kuala', 'location'), ('Lumpur', 'location'), ('Sempena', 'OTHER'), ('sambutan', 'OTHER'), ('Aidilfitri', 'event'), ('minggu', 'time'), ('depan', 'time'), ('Perdana', 'person'), ('Menteri', 'person'), ('Tun', 'person'), ('Dr', 'person'), ('Mahathir', 'person'), ('Mohamad', 'person'), ('dan', 'OTHER'), ('Menteri', 'person'), ('Pengangkutan', 'person'), ('Anthony', 'person'), ('Loke', 'person'), ('Siew', 'person'), ('Fook', 'person'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'OTHER'), ('masing-masing', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'organization'), ('Keselamatan', 'organization'), ('Jalan', 'organization'), ('Raya', 'organization'), ('(Jkjr)', 'organization'), ('itu', 'OTHER'), ('Dr', 'person'), ('Mahathir', 'person'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  17%|█▋        | 4348/25761 [27:13<2:15:00,  2.64it/s, accuracy=0.999, cost=0.0497]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  32%|███▏      | 8362/25761 [52:15<1:45:21,  2.75it/s, accuracy=1, cost=0.0821]    IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 25761/25761 [2:41:02<00:00,  2.67it/s, accuracy=1, cost=6.24e-5]     \n",
      "test minibatch loop: 100%|██████████| 4595/4595 [11:40<00:00,  6.56it/s, accuracy=1, cost=0.0933]    \n",
      "train minibatch loop:   0%|          | 0/25761 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 10362.454544067383\n",
      "epoch: 2, training loss: 0.032517, training acc: 0.999837, valid loss: 4.019222, valid acc: 0.988776\n",
      "\n",
      "[('Kuala', 'location'), ('Lumpur', 'location'), ('Sempena', 'OTHER'), ('sambutan', 'OTHER'), ('Aidilfitri', 'event'), ('minggu', 'time'), ('depan', 'time'), ('Perdana', 'person'), ('Menteri', 'person'), ('Tun', 'person'), ('Dr', 'person'), ('Mahathir', 'person'), ('Mohamad', 'person'), ('dan', 'OTHER'), ('Menteri', 'person'), ('Pengangkutan', 'person'), ('Anthony', 'person'), ('Loke', 'person'), ('Siew', 'person'), ('Fook', 'person'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'OTHER'), ('masing-masing', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'organization'), ('Keselamatan', 'organization'), ('Jalan', 'organization'), ('Raya', 'organization'), ('(Jkjr)', 'organization'), ('itu', 'OTHER'), ('Dr', 'person'), ('Mahathir', 'person'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:  88%|████████▊ | 22772/25761 [2:22:03<19:50,  2.51it/s, accuracy=1, cost=-.000227]    IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 25761/25761 [2:40:40<00:00,  2.67it/s, accuracy=1, cost=0.000437]  \n",
      "test minibatch loop:  16%|█▋        | 753/4595 [01:55<10:06,  6.33it/s, accuracy=0.999, cost=0.193] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "test minibatch loop: 100%|██████████| 4595/4595 [11:39<00:00,  6.57it/s, accuracy=1, cost=0.00072]   \n",
      "train minibatch loop:   0%|          | 0/25761 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 10339.672159671783\n",
      "epoch: 3, training loss: 0.025209, training acc: 0.999874, valid loss: 4.749696, valid acc: 0.988282\n",
      "\n",
      "[('Kuala', 'location'), ('Lumpur', 'location'), ('Sempena', 'OTHER'), ('sambutan', 'OTHER'), ('Aidilfitri', 'event'), ('minggu', 'time'), ('depan', 'time'), ('Perdana', 'person'), ('Menteri', 'person'), ('Tun', 'person'), ('Dr', 'person'), ('Mahathir', 'person'), ('Mohamad', 'person'), ('dan', 'OTHER'), ('Menteri', 'organization'), ('Pengangkutan', 'organization'), ('Anthony', 'person'), ('Loke', 'person'), ('Siew', 'person'), ('Fook', 'person'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'OTHER'), ('masing-masing', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'organization'), ('Keselamatan', 'organization'), ('Jalan', 'organization'), ('Raya', 'organization'), ('(Jkjr)', 'organization'), ('itu', 'OTHER'), ('Dr', 'person'), ('Mahathir', 'person'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:   0%|          | 113/25761 [00:43<2:39:10,  2.69it/s, accuracy=0.999, cost=0.214]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  16%|█▌        | 4083/25761 [25:31<2:26:29,  2.47it/s, accuracy=1, cost=0.000936]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  31%|███       | 7946/25761 [49:35<1:45:46,  2.81it/s, accuracy=1, cost=0.0037]    IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  46%|████▌     | 11865/25761 [1:14:03<1:22:56,  2.79it/s, accuracy=0.999, cost=0.058] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  61%|██████    | 15699/25761 [1:38:00<1:01:16,  2.74it/s, accuracy=1, cost=0.00297]   IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  76%|███████▋  | 19649/25761 [2:02:44<36:27,  2.79it/s, accuracy=1, cost=0.00105]   IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop:  92%|█████████▏| 23581/25761 [2:27:16<12:54,  2.82it/s, accuracy=1, cost=0.0141]    IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "test minibatch loop:  36%|███▌      | 1664/4595 [04:15<07:04,  6.90it/s, accuracy=0.954, cost=34.9]  IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "test minibatch loop: 100%|██████████| 4595/4595 [11:39<00:00,  6.57it/s, accuracy=1, cost=0.0652]    "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 10351.633486032486\n",
      "epoch: 4, training loss: 0.021230, training acc: 0.999892, valid loss: 3.032607, valid acc: 0.991644\n",
      "\n",
      "[('Kuala', 'organization'), ('Lumpur', 'location'), ('Sempena', 'OTHER'), ('sambutan', 'OTHER'), ('Aidilfitri', 'event'), ('minggu', 'time'), ('depan', 'time'), ('Perdana', 'person'), ('Menteri', 'person'), ('Tun', 'person'), ('Dr', 'person'), ('Mahathir', 'person'), ('Mohamad', 'person'), ('dan', 'OTHER'), ('Menteri', 'organization'), ('Pengangkutan', 'organization'), ('Anthony', 'person'), ('Loke', 'person'), ('Siew', 'person'), ('Fook', 'person'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'OTHER'), ('masing-masing', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'organization'), ('Keselamatan', 'organization'), ('Jalan', 'organization'), ('Raya', 'organization'), ('(Jkjr)', 'organization'), ('itu', 'OTHER'), ('Dr', 'person'), ('Mahathir', 'person'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "for e in range(epoch):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss = [], [], [], []\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        batch_y = train_Y[i : index]\n",
    "        batch_masks = train_masks[i : index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_y = pad_sequences(batch_y, padding='post')\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post')\n",
    "        \n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.Y: batch_y,\n",
    "                model.MASK: batch_masks,\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i : index]\n",
    "        batch_y = test_Y[i : index]\n",
    "        batch_masks = test_masks[i : index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_y = pad_sequences(batch_y, padding='post')\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post')\n",
    "        \n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.Y: batch_y,\n",
    "                model.MASK: batch_masks,\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    train_loss = np.mean(train_loss)\n",
    "    train_acc = np.mean(train_acc)\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_acc = np.mean(test_acc)\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (e, train_loss, train_acc, test_loss, test_acc)\n",
    "    )\n",
    "    predicted = sess.run(model.tags_seq,\n",
    "                feed_dict = {\n",
    "                    model.X: [parsed_sequence],\n",
    "                    model.MASK: [input_mask]\n",
    "                },\n",
    "        )[0]\n",
    "    merged = merge_sentencepiece_tokens_tagging(bert_sequence, [idx2tag[d] for d in predicted])\n",
    "    print(list(zip(merged[0], merged[1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bert-base-entities/model.ckpt'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'bert-base-entities/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-entities/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = len(tag2idx)\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate,\n",
    "    training = False\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'bert-base-entities/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop:  65%|██████▌   | 2999/4595 [07:39<04:24,  6.04it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_X), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i : index]\n",
    "    batch_y = test_Y[i : index]\n",
    "    batch_masks = test_masks[i : index]\n",
    "    batch_x = pad_sequences(batch_x, padding='post')\n",
    "    batch_y = pad_sequences(batch_y, padding='post')\n",
    "    batch_masks = pad_sequences(batch_masks, padding='post')\n",
    "    predicted = pred2label(sess.run(model.tags_seq,\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.MASK: batch_masks,\n",
    "            },\n",
    "    ))\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       OTHER    0.99224   0.99931   0.99576   5160854\n",
      "         PAD    1.00000   1.00000   1.00000    877767\n",
      "           X    0.99995   1.00000   0.99998   2921053\n",
      "       event    0.99911   0.88679   0.93961    143787\n",
      "         law    0.99704   0.97040   0.98354    146950\n",
      "    location    0.98677   0.98420   0.98548    428869\n",
      "organization    0.99335   0.95355   0.97304    694150\n",
      "      person    0.97636   0.99476   0.98547    507960\n",
      "    quantity    0.99965   0.99803   0.99884     88200\n",
      "        time    0.98462   0.99938   0.99194    179880\n",
      "\n",
      "    accuracy                        0.99406  11149470\n",
      "   macro avg    0.99291   0.97864   0.98537  11149470\n",
      "weighted avg    0.99409   0.99406   0.99400  11149470\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/token_type_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'bert/embeddings/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/query/bias',\n",
       " 'bert/encoder/layer_4/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/key/bias',\n",
       " 'bert/encoder/layer_4/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/value/bias',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax',\n",
       " 'bert/encoder/layer_4/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_4/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_4/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_4/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/output/dense/bias',\n",
       " 'bert/encoder/layer_4/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/query/bias',\n",
       " 'bert/encoder/layer_5/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/key/bias',\n",
       " 'bert/encoder/layer_5/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/value/bias',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax',\n",
       " 'bert/encoder/layer_5/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_5/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_5/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_5/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/output/dense/bias',\n",
       " 'bert/encoder/layer_5/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_6/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/query/bias',\n",
       " 'bert/encoder/layer_6/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/key/bias',\n",
       " 'bert/encoder/layer_6/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/value/bias',\n",
       " 'bert/encoder/layer_6/attention/self/Softmax',\n",
       " 'bert/encoder/layer_6/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_6/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_6/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_6/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_6/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_6/output/dense/kernel',\n",
       " 'bert/encoder/layer_6/output/dense/bias',\n",
       " 'bert/encoder/layer_6/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_7/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/query/bias',\n",
       " 'bert/encoder/layer_7/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/key/bias',\n",
       " 'bert/encoder/layer_7/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/value/bias',\n",
       " 'bert/encoder/layer_7/attention/self/Softmax',\n",
       " 'bert/encoder/layer_7/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_7/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_7/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_7/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_7/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_7/output/dense/kernel',\n",
       " 'bert/encoder/layer_7/output/dense/bias',\n",
       " 'bert/encoder/layer_7/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_8/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/query/bias',\n",
       " 'bert/encoder/layer_8/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/key/bias',\n",
       " 'bert/encoder/layer_8/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/value/bias',\n",
       " 'bert/encoder/layer_8/attention/self/Softmax',\n",
       " 'bert/encoder/layer_8/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_8/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_8/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_8/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_8/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_8/output/dense/kernel',\n",
       " 'bert/encoder/layer_8/output/dense/bias',\n",
       " 'bert/encoder/layer_8/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_9/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/query/bias',\n",
       " 'bert/encoder/layer_9/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/key/bias',\n",
       " 'bert/encoder/layer_9/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/value/bias',\n",
       " 'bert/encoder/layer_9/attention/self/Softmax',\n",
       " 'bert/encoder/layer_9/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_9/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_9/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_9/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_9/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_9/output/dense/kernel',\n",
       " 'bert/encoder/layer_9/output/dense/bias',\n",
       " 'bert/encoder/layer_9/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_10/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/query/bias',\n",
       " 'bert/encoder/layer_10/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/key/bias',\n",
       " 'bert/encoder/layer_10/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/value/bias',\n",
       " 'bert/encoder/layer_10/attention/self/Softmax',\n",
       " 'bert/encoder/layer_10/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_10/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_10/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_10/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_10/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_10/output/dense/kernel',\n",
       " 'bert/encoder/layer_10/output/dense/bias',\n",
       " 'bert/encoder/layer_10/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_11/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/query/bias',\n",
       " 'bert/encoder/layer_11/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/key/bias',\n",
       " 'bert/encoder/layer_11/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/value/bias',\n",
       " 'bert/encoder/layer_11/attention/self/Softmax',\n",
       " 'bert/encoder/layer_11/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_11/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_11/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_11/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_11/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_11/output/dense/kernel',\n",
       " 'bert/encoder/layer_11/output/dense/bias',\n",
       " 'bert/encoder/layer_11/output/LayerNorm/gamma',\n",
       " 'bert/pooler/dense/kernel',\n",
       " 'bert/pooler/dense/bias',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'dense_1/kernel',\n",
       " 'dense_1/bias',\n",
       " 'transitions',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1',\n",
       " 'logits']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-entities/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-36-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 204 variables.\n",
      "INFO:tensorflow:Converted 204 variables to const ops.\n",
      "10187 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('bert-base-entities', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph\n",
    "\n",
    "g = load_graph('bert-base-entities/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "mask = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'Kyrgios, 25, membuat pesanan itu kerana menyedaari pelbagai kesukaran menimpa rakyat Australia ekoran perintah kawalan pergerakan yang diumumkan Mac lalu bagi memerangi wabak COVID-19 di negara berkenaan. Pemain tenis ranking ke-40 dunia yang dilahirkan di Canberra itu meminta pengikut dan penyokongnya agar jangan tidur dalam keadaan perut kosong dalam hantaran Instagram yang meraih lebih 92,000 tanda suka.'\n",
    "\n",
    "ori, sequence = entities_textcleaning(string)\n",
    "parsed_sequence, bert_sequence, input_mask = parse_X(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Kyrgios', 'organization'), ('25', 'time'), ('membuat', 'OTHER'), ('pesanan', 'OTHER'), ('itu', 'OTHER'), ('kerana', 'OTHER'), ('menyedaari', 'OTHER'), ('pelbagai', 'OTHER'), ('kesukaran', 'OTHER'), ('menimpa', 'OTHER'), ('rakyat', 'OTHER'), ('Australia', 'location'), ('ekoran', 'OTHER'), ('perintah', 'OTHER'), ('kawalan', 'OTHER'), ('pergerakan', 'OTHER'), ('yang', 'OTHER'), ('diumumkan', 'OTHER'), ('Mac', 'time'), ('lalu', 'time'), ('bagi', 'OTHER'), ('memerangi', 'OTHER'), ('wabak', 'OTHER'), ('Covid-19', 'OTHER'), ('di', 'OTHER'), ('negara', 'OTHER'), ('berkenaan', 'OTHER'), ('Pemain', 'OTHER'), ('tenis', 'OTHER'), ('ranking', 'OTHER'), ('ke-40', 'OTHER'), ('dunia', 'OTHER'), ('yang', 'OTHER'), ('dilahirkan', 'OTHER'), ('di', 'OTHER'), ('Canberra', 'location'), ('itu', 'OTHER'), ('meminta', 'OTHER'), ('pengikut', 'OTHER'), ('dan', 'OTHER'), ('penyokongnya', 'OTHER'), ('agar', 'OTHER'), ('jangan', 'OTHER'), ('tidur', 'OTHER'), ('dalam', 'OTHER'), ('keadaan', 'OTHER'), ('perut', 'OTHER'), ('kosong', 'OTHER'), ('dalam', 'OTHER'), ('hantaran', 'OTHER'), ('Instagram', 'OTHER'), ('yang', 'OTHER'), ('meraih', 'OTHER'), ('lebih', 'OTHER'), ('92', 'OTHER'), ('000', 'OTHER'), ('tanda', 'OTHER'), ('suka', 'OTHER')]\n"
     ]
    }
   ],
   "source": [
    "predicted = test_sess.run(logits,\n",
    "            feed_dict = {\n",
    "                x: [parsed_sequence],\n",
    "                mask: [input_mask]\n",
    "            },\n",
    "    )[0]\n",
    "merged = merge_sentencepiece_tokens_tagging(bert_sequence, [idx2tag[d] for d in predicted])\n",
    "print(list(zip(merged[0], merged[1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'huseinhouse-storage'\n",
    "Key = 'bert-base-entities/frozen_model.pb'\n",
    "outPutname = \"v34/entity/bert-base-entity.pb\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
