{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams = {'vocab_size': 32000, 'do_lower_case': False, 'vocab_file': 'bahasa.wordpiece',\n",
    "          'pretrain_tfrecords': 'dataset/pretrain_data.tfrecord',\n",
    "          'model_size': 'base', 'embedding_size': 768, 'max_seq_length': 512,\n",
    "           \n",
    "    #   self.learning_rate = 2e-4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import configure_pretraining\n",
    "from util import training_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name': 'electra-small',\n",
       " 'debug': False,\n",
       " 'do_train': True,\n",
       " 'do_eval': False,\n",
       " 'electra_objective': True,\n",
       " 'gen_weight': 1.0,\n",
       " 'disc_weight': 50.0,\n",
       " 'mask_prob': 0.15,\n",
       " 'learning_rate': 0.0005,\n",
       " 'lr_decay_power': 1.0,\n",
       " 'weight_decay_rate': 0.01,\n",
       " 'num_warmup_steps': 10000,\n",
       " 'iterations_per_loop': 200,\n",
       " 'save_checkpoints_steps': 1000,\n",
       " 'num_train_steps': 1000000,\n",
       " 'num_eval_steps': 100,\n",
       " 'model_size': 'base',\n",
       " 'model_hparam_overrides': {},\n",
       " 'embedding_size': 768,\n",
       " 'vocab_size': 32000,\n",
       " 'do_lower_case': False,\n",
       " 'uniform_generator': False,\n",
       " 'untied_generator_embeddings': False,\n",
       " 'untied_generator': True,\n",
       " 'generator_layers': 1.0,\n",
       " 'generator_hidden_size': 0.25,\n",
       " 'disallow_correct': False,\n",
       " 'temperature': 1.0,\n",
       " 'max_seq_length': 128,\n",
       " 'train_batch_size': 128,\n",
       " 'eval_batch_size': 128,\n",
       " 'use_tpu': False,\n",
       " 'num_tpu_cores': 1,\n",
       " 'tpu_job_name': None,\n",
       " 'tpu_name': None,\n",
       " 'tpu_zone': None,\n",
       " 'gcp_project': None,\n",
       " 'pretrain_tfrecords': 'dataset/pretrain_data.tfrecord',\n",
       " 'vocab_file': 'bahasa.wordpiece',\n",
       " 'model_dir': 'dataset/models/electra-small',\n",
       " 'results_txt': 'dataset/models/electra-small/results/unsup_results.txt',\n",
       " 'results_pkl': 'dataset/models/electra-small/results/unsup_results.pkl',\n",
       " 'max_predictions_per_seq': 19}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = configure_pretraining.PretrainingConfig('electra-small', 'dataset', **hparams)\n",
    "config.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vocab_size': 32000,\n",
       " 'hidden_size': 768,\n",
       " 'num_hidden_layers': 12,\n",
       " 'num_attention_heads': 12,\n",
       " 'hidden_act': 'gelu',\n",
       " 'intermediate_size': 3072,\n",
       " 'hidden_dropout_prob': 0.1,\n",
       " 'attention_probs_dropout_prob': 0.1,\n",
       " 'max_position_embeddings': 512,\n",
       " 'type_vocab_size': 2,\n",
       " 'initializer_range': 0.02}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_config = training_utils.get_bert_config(config)\n",
    "bert_config.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('SMALL-config.json', 'w') as fopen:\n",
    "    json.dump(hparams, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
