{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import sentencepiece as spm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('tiny-bert-v1/sp10m.cased.bert.model')\n",
    "\n",
    "with open('tiny-bert-v1/sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v):\n",
    "        self.vocab = v\n",
    "        pass\n",
    "    \n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(sp_model, string, return_unicode=False, sample=False)\n",
    "    \n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [sp_model.PieceToId(piece) for piece in tokens]\n",
    "    \n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [sp_model.IdToPiece(i) for i in ids]\n",
    "    \n",
    "tokenizer = Tokenizer(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "from unidecode import unidecode\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'tiny-bert-v1/model.ckpt'\n",
    "BERT_CONFIG = 'tiny-bert-v1/config.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rules import normalized_chars\n",
    "import random\n",
    "\n",
    "laughing = {\n",
    "    'huhu',\n",
    "    'haha',\n",
    "    'gagaga',\n",
    "    'hihi',\n",
    "    'wkawka',\n",
    "    'wkwk',\n",
    "    'kiki',\n",
    "    'keke',\n",
    "    'huehue',\n",
    "    'hshs',\n",
    "    'hoho',\n",
    "    'hewhew',\n",
    "    'uwu',\n",
    "    'sksk',\n",
    "    'ksks',\n",
    "    'gituu',\n",
    "    'gitu',\n",
    "    'mmeeooww',\n",
    "    'meow',\n",
    "    'alhamdulillah',\n",
    "    'muah',\n",
    "    'mmuahh',\n",
    "    'hehe',\n",
    "    'salamramadhan',\n",
    "    'happywomensday',\n",
    "    'jahagaha',\n",
    "    'ahakss',\n",
    "    'ahksk'\n",
    "}\n",
    "\n",
    "def make_cleaning(s, c_dict):\n",
    "    s = s.translate(c_dict)\n",
    "    return s\n",
    "\n",
    "def cleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    \n",
    "    string = ' '.join(\n",
    "        [make_cleaning(w, normalized_chars) for w in string.split()]\n",
    "    )\n",
    "    string = re.sub('\\(dot\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    \n",
    "    chars = '.,/'\n",
    "    for c in chars:\n",
    "        string = string.replace(c, f' {c} ')\n",
    "        \n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    x = []\n",
    "    for word in string:\n",
    "        word = word.lower()\n",
    "        if any([laugh in word for laugh in laughing]):\n",
    "            if random.random() >= 0.5:\n",
    "                x.append(word)\n",
    "        else:\n",
    "            x.append(word)\n",
    "    string = [w.title() if w[0].isupper() else w for w in x]\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../Malaya-Dataset/news-sentiment/sentiment-data-v2.csv')\n",
    "Y = LabelEncoder().fit_transform(df.label)\n",
    "\n",
    "with open('../Malaya-Dataset/polarity/polarity-negative-translated.txt','r') as fopen:\n",
    "    texts = fopen.read().split('\\n')\n",
    "labels = [0] * len(texts)\n",
    "\n",
    "with open('../Malaya-Dataset/polarity/polarity-positive-translated.txt','r') as fopen:\n",
    "    positive_texts = fopen.read().split('\\n')\n",
    "labels += [1] * len(positive_texts)\n",
    "texts += positive_texts\n",
    "texts += df.iloc[:,1].tolist()\n",
    "labels += Y.tolist()\n",
    "\n",
    "assert len(labels) == len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('../Malaya-Dataset/multidomain-sentiment/bm-amazon.json') as fopen:\n",
    "    amazon = json.load(fopen)\n",
    "    \n",
    "with open('../Malaya-Dataset/multidomain-sentiment/bm-imdb.json') as fopen:\n",
    "    imdb = json.load(fopen)\n",
    "    \n",
    "with open('../Malaya-Dataset/multidomain-sentiment/bm-yelp.json') as fopen:\n",
    "    yelp = json.load(fopen)\n",
    "    \n",
    "texts += amazon['negative']\n",
    "labels += [0] * len(amazon['negative'])\n",
    "texts += amazon['positive']\n",
    "labels += [1] * len(amazon['positive'])\n",
    "\n",
    "texts += imdb['negative']\n",
    "labels += [0] * len(imdb['negative'])\n",
    "texts += imdb['positive']\n",
    "labels += [1] * len(imdb['positive'])\n",
    "\n",
    "texts += yelp['negative']\n",
    "labels += [0] * len(yelp['negative'])\n",
    "texts += yelp['positive']\n",
    "labels += [1] * len(yelp['positive'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../sentiment/strong-positives.json') as fopen:\n",
    "    positives = json.load(fopen)\n",
    "    positives = random.sample(positives, 500000)\n",
    "    \n",
    "len(positives)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('../sentiment/strong-negatives.json') as fopen:\n",
    "    negatives = json.load(fopen)\n",
    "    negatives = random.sample(negatives, 500000)\n",
    "    \n",
    "len(negatives)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts += negatives\n",
    "labels += [0] * len(negatives)\n",
    "texts += positives\n",
    "labels += [1] * len(positives)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1016725/1016725 [01:11<00:00, 14277.50it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "    texts[i] = cleaning(texts[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1016725/1016725 [00:00<00:00, 1395409.49it/s]\n"
     ]
    }
   ],
   "source": [
    "actual_t, actual_l = [], []\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "    if len(texts[i]) > 2:\n",
    "        actual_t.append(texts[i])\n",
    "        actual_l.append(labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1016724/1016724 [01:41<00:00, 10049.98it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "input_ids, input_masks = [], []\n",
    "\n",
    "for text in tqdm(actual_t):\n",
    "    tokens_a = tokenizer.tokenize(text)\n",
    "    tokens = [\"[CLS]\"] + tokens_a + [\"[SEP]\"]\n",
    "    input_id = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    input_mask = [1] * len(input_id)\n",
    "    \n",
    "    input_ids.append(input_id)\n",
    "    input_masks.append(input_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 2\n",
    "batch_size = 256\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(texts) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_initializer(initializer_range=0.02):\n",
    "    return tf.truncated_normal_initializer(stddev=initializer_range)\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 2e-5,\n",
    "        training = True,\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.MASK = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.MASK,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        output_layer = tf.layers.dense(\n",
    "            output_layer,\n",
    "            bert_config.hidden_size,\n",
    "            activation=tf.tanh,\n",
    "            kernel_initializer=create_initializer())\n",
    "        self.logits_seq = tf.layers.dense(output_layer, dimension_output,\n",
    "                                         kernel_initializer=create_initializer())\n",
    "        self.logits_seq = tf.identity(self.logits_seq, name = 'logits_seq')\n",
    "        self.logits = self.logits_seq[:, 0]\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        correct_pred = tf.equal(\n",
    "            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n",
    "        )\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:409: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Restoring parameters from tiny-bert-v1/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 2\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_input_ids, test_input_ids, train_Y, test_Y, train_mask, test_mask = train_test_split(\n",
    "    input_ids, actual_l, input_masks, test_size = 0.2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 3178/3178 [09:07<00:00,  5.80it/s, accuracy=1, cost=0.0048]    \n",
      "test minibatch loop: 100%|██████████| 795/795 [00:57<00:00, 13.81it/s, accuracy=0.988, cost=0.0593]\n",
      "train minibatch loop:   0%|          | 0/3178 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.124446, training acc: 0.948765, valid loss: 0.054366, valid acc: 0.980876\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 3178/3178 [09:06<00:00,  5.81it/s, accuracy=1, cost=0.002]      \n",
      "test minibatch loop: 100%|██████████| 795/795 [00:57<00:00, 13.82it/s, accuracy=0.988, cost=0.0579] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.040027, training acc: 0.985326, valid loss: 0.041185, valid acc: 0.985136\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import time\n",
    "\n",
    "for EPOCH in range(epoch):\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = [], [], [], []\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_input_ids), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_input_ids))\n",
    "        batch_x = train_input_ids[i: index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_mask = train_mask[i: index]\n",
    "        batch_mask = pad_sequences(batch_mask, padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.MASK: batch_mask\n",
    "            },\n",
    "        )\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "        \n",
    "    pbar = tqdm(range(0, len(test_input_ids), batch_size), desc = 'test minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_input_ids))\n",
    "        batch_x = test_input_ids[i: index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_mask = test_mask[i: index]\n",
    "        batch_mask = pad_sequences(batch_mask, padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.Y: batch_y,\n",
    "                model.X: batch_x,\n",
    "                model.MASK: batch_mask\n",
    "            },\n",
    "        )\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "        \n",
    "    train_loss = np.mean(train_loss)\n",
    "    train_acc = np.mean(train_acc)\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_acc = np.mean(test_acc)\n",
    "    \n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (EPOCH, train_loss, train_acc, test_loss, test_acc)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tiny-bert-sentiment/model.ckpt'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'tiny-bert-sentiment/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-sentiment/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = 2\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate,\n",
    "    training = False\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'tiny-bert-sentiment/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/token_type_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'bert/embeddings/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/pooler/dense/kernel',\n",
       " 'bert/pooler/dense/bias',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'dense_1/kernel',\n",
       " 'dense_1/bias',\n",
       " 'logits_seq',\n",
       " 'logits',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 795/795 [00:46<00:00, 17.04it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_input_ids), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_input_ids))\n",
    "    batch_x = test_input_ids[i: index]\n",
    "    batch_x = pad_sequences(batch_x, padding='post')\n",
    "    batch_mask = test_mask[i: index]\n",
    "    batch_mask = pad_sequences(batch_mask, padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    \n",
    "    predict_Y += np.argmax(sess.run(model.logits,\n",
    "            feed_dict = {\n",
    "            model.X: batch_x,\n",
    "            model.MASK: batch_mask\n",
    "            },\n",
    "    ), 1, ).tolist()\n",
    "    real_Y += batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative    0.98953   0.98579   0.98766    101153\n",
      "    positive    0.98599   0.98968   0.98783    102192\n",
      "\n",
      "    accuracy                        0.98774    203345\n",
      "   macro avg    0.98776   0.98774   0.98774    203345\n",
      "weighted avg    0.98775   0.98774   0.98774    203345\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, predict_Y, target_names = ['negative', 'positive'], digits = 5\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-sentiment/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-27-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 75 variables.\n",
      "INFO:tensorflow:Converted 75 variables to const ops.\n",
      "2456 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('tiny-bert-sentiment', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[4.4864928e-04, 9.9955136e-01]], dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = load_graph('tiny-bert-sentiment/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "mask = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)\n",
    "result = test_sess.run(tf.nn.softmax(logits), feed_dict = {x: [input_id], mask: [input_mask]})\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "bucketName = 'huseinhouse-storage'\n",
    "Key = 'tiny-bert-sentiment/frozen_model.pb'\n",
    "outPutname = \"v34/sentiment/tiny-bert-sentiment.pb\"\n",
    "\n",
    "s3 = boto3.client('s3')\n",
    "s3.upload_file(Key,bucketName,outPutname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
