{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1,2,3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "\n",
    "def nq_dataset_fn(split, shuffle_files=False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(['cnn-summarization.tsv'])\n",
    "\n",
    "    ds = ds.map(\n",
    "      functools.partial(tf.io.decode_csv, record_defaults=[\"\", \"\"],\n",
    "                        field_delim=\"\\t\", use_quote_delim=False),\n",
    "      num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "    # Map each tuple to a {\"question\": ... \"answer\": ...} dict.\n",
    "    ds = ds.map(lambda *ex: dict(zip([\"question\", \"answer\"], ex)))\n",
    "    return ds\n",
    "    \n",
    "def trivia_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        \"\"\"Map {\"question\": ..., \"answer\": ...}->{\"inputs\": ..., \"targets\": ...}.\"\"\"\n",
    "        return {\n",
    "            \"inputs\":\n",
    "                 tf.strings.join(\n",
    "                     [\"ringkasan: \", ex[\"question\"]]),\n",
    "            \"targets\": ex[\"answer\"]\n",
    "        }\n",
    "    return ds.map(to_inputs_and_targets, \n",
    "                    num_parallel_calls=tf.data.experimental.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "t5.data.TaskRegistry.remove('cnn_summary')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'cnn_summary',\n",
    "    dataset_fn=nq_dataset_fn,\n",
    "    splits=[\"train\"],\n",
    "    text_preprocessor=[trivia_preprocessor],\n",
    "    sentencepiece_model_path='sp10m.cased.t5.model',\n",
    "    postprocess_fn=t5.data.postprocessors.lower_text, \n",
    "    metric_fns=[t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nq_task = t5.data.TaskRegistry.get(\"cnn_summary\")\n",
    "ds = nq_task.get_dataset(split=\"train\", sequence_length={\"inputs\": 1024, \"targets\": 256})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b\"ringkasan: (CNN) - Edward Dolman adalah ketua pegawai eksekutif pertama Christie untuk menjadi seorang pakar, yang memahami tekanan dan sifat menilai seni. Sebahagian daripada misinya menarik perhatian pakar yang bermotivasi tinggi ke dalam perniagaan mereka. Seorang pekerja Christie berdiri di samping lukisan di rumah lelongan Christie di London pada 4 Julai. Dolman dilahirkan di London pada tahun 1960 dan menghadiri Kolej Dulwich dari 1971-1978, sebelum belajar Sejarah Seni di Southampton University. Beliau kemudiannya melengkapkan ijazah di Pusat Pengajian untuk Fine and Decorative Arts, yang mempunyai hubungan erat dengan Victoria dan Albert Museum. Tidak menjadi kelahiran darah biru, dia menyertai Christie kerana hubungannya dengan Christie'S South Kensington, bermula sebagai porter di jabatan perabot. Selepas setahun dia memegang peranan pakar dalam jabatan perabot di South Kensington, di mana dia akan melihat segala-galanya dari perabot Eropah ke China dalam saluangom tinggi dalam tempoh 10 tahun akan datang. Sementara itu, beliau menjadi pelelong, melakukan beberapa jualan luar tapak di estet negara dan perjalanan ke Perancis dan Belgium untuk menghasilkan perniagaan di pasaran pertengahan perabot. Pada masa itu dia berada di Jawatankuasa Pengurusan, dan South Kensington mula melakukan jualan yang hebat. Akhirnya pada tahun 1995, beliau memutuskan untuk menjadi seorang pengurus dan mengambil kursus di Institut Pengarah di Mall untuk mempelajari bahasa perniagaan dan motivasi orang. Dolman kemudiannya menjadi Pengarah Urusan Christie'S Amsterdam, menerapkan teknik dan pemasaran baru untuk meningkatkan minat dalam seni berkualiti tinggi. Dua puluh bulan kemudian, dia kembali ke Christie'S London di King Street sebagai Pengarah Komersial untuk menjalankan Christie'S Europe. Semasa beliau cuba mengembangkan kategori baru untuk memenangi perniagaan, beliau telah dijadikan Pengarah Urusan. Pada tahun 1999, Dolman berpindah ke New York sebagai Pengarah Urusan Antarabangsa dan kemudian sebagai Ketua Eksekutif. Bersama dengan Kumpulan Pengurusan Antarabangsa Christie, mereka mengembangkan rancangan perniagaan lima tahun yang intensif, dan Dolman mengambil alih dari Christopher Davidge, seperti yang diluluskan oleh Francois Pinault.\", 'inputs': array([ 4140,    36,    50,    15,     4, 29161,     5,    15,     7,\n",
      "        6493,  8974,   463,    35,  1103,   600,  4337,   131, 13293,\n",
      "        1798,    21,    56,   105,  1598,    14,    17,  1859,  1786,\n",
      "          16,  2087,  2471,  1177,     3,  5565,    47,  2318,    26,\n",
      "         717,  1089,  1598,    17,   181,  1538,   765, 10610,   182,\n",
      "          30,    19,   622,    45,     3,  1952,   587, 13293,  1798,\n",
      "        3792,    18,  1101,  5727,    18,   160, 26752, 13293,  1798,\n",
      "          18,  1708,    23,   172,   733,     3,  8974,   463,  1820,\n",
      "          18,  1708,    23,    31,  3200,    16,  2003,  2971,   411,\n",
      "         783,  1641,  1128,    37,  5161,     7, 23525,    14,   137,\n",
      "        1738,  2894,  2946,    18, 28353,  1920,     3,    99,  1072,\n",
      "        5987,  4142,    18,   609,  2521,    21,   553,  2189,    63,\n",
      "        1092,  7225,  7780, 15406,    14,    17,    85,   684,  7574,\n",
      "          22,  5900,    16,  8455, 17096,     3,  1147,    56,  2825,\n",
      "        1867,  4999,    14,    18,    13,   795, 13293,  1798,    69,\n",
      "       14342,    22, 13293,  1798,    12,   237,  3000, 25476,    14,\n",
      "         348,    41, 12087,   254,    18,   266, 11717,     3,   799,\n",
      "        1305,    18,    13,  1498,   772,  1598,    19,   266, 11717,\n",
      "          18,  3000, 25476,    14,    18,   119,    18,    13,    38,\n",
      "         522,  1073,     7, 14211,    37, 11717,   838,    30,   303,\n",
      "          19,  7827,  8461,  2462,   182,    19,   274,   250,    31,\n",
      "          38,   619,     3,   565,    24,    14,    71,    56, 17042,\n",
      "        3526,    14,   341,   103,  1930,   236,  1376,    18, 15485,\n",
      "          48,    16,  1314,    30,   204,    16,  5272,    21,   803,\n",
      "         622,    18,   623,  2149, 11717,     3,    67,    97,    24,\n",
      "          18,    13,   351,    18,   895,  1881,    14,    16,  3000,\n",
      "       25476,   407,   341,  1930,    17,  2715,     3,  3880,    23,\n",
      "          31, 12494,    71,  1609,    21,    56,   105,  3504,    16,\n",
      "         187,  2837,    18,  1700,   718,    18,  7898,    21,  4539,\n",
      "         230,   622,    16,  9469,    53,     3,  8974,   463,  1072,\n",
      "          56,   718,  5773, 13293,  1798,    12,   237, 14228,    14,\n",
      "        6723,  3319,    16,  5457,   163,    21,   340,  2770,    19,\n",
      "        1177,  3212,   182,     3,  1698,  2888,   235,   346,    14,\n",
      "          18,    13,   448,    30, 13293,  1798,    12,   237,  1708,\n",
      "          18,  3400,  4401,    41,   718, 17326,    21,   779, 13293,\n",
      "        1798,    12,   237, 16191,     3,  1835,    71,  1006,  3112,\n",
      "        1504,   163,    21,  1053,   622,    14,    71,    33,  1362,\n",
      "         718,  5773,     3,    67,    31,  8725,  8974,   463,  2173,\n",
      "          30,   743,  2070,    41,   718,  5773,   843,    16,   346,\n",
      "          41,   202,  1828,     3,  2712,    22,   971,  1881,   843,\n",
      "       13293,  1798,    14,    45,  3112,  1269,   622,   487,    31,\n",
      "          17, 13895,    14,    16,  8974,   463,   187,  1510,    37,\n",
      "       13970,  3449,  1174,    14,    59,    17,  1780,    32, 22521,\n",
      "        6433, 12133,     3,     1]), 'targets_plaintext': b\"Dolman adalah pakar dalam jabatan perabot South Kensington selama 10 tahun. Pada tahun 1996 beliau telah dijadikan Pengarah Urusan Christie'S Amsterdam. Rancangan perniagaan lima tahun dengan Christie'S International Management Group\", 'targets': array([ 8974,   463,    35,  1598,    19,   266, 11717,  3000, 25476,\n",
      "         331,   250,    31,     3,    67,    31,  3063,    71,    33,\n",
      "        1362,   718,  5773, 13293,  1798,    12,   237, 14228,     3,\n",
      "        2562,   622,   487,    31,    22, 13293,  1798,    12,   237,\n",
      "        1595,  7321,  2959,     1])}\n"
     ]
    }
   ],
   "source": [
    "for ex in tfds.as_numpy(ds.take(1)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "t5.data.MixtureRegistry.remove(\"summary_bahasa\")\n",
    "t5.data.MixtureRegistry.add(\n",
    "    \"summary_bahasa\",\n",
    "    [\"cnn_summary\"],\n",
    "     default_rate=1.0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import gin\n",
    "# gin.parse_config_file('pretrained_models_base_operative_config.gin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = t5.models.MtfModel(\n",
    "    model_dir='t5-base-summary',\n",
    "    tpu=None,\n",
    "    tpu_topology=None,\n",
    "    model_parallelism=2,\n",
    "    batch_size=8,\n",
    "    sequence_length={\"inputs\": 1024, \"targets\": 256},\n",
    "    learning_rate_schedule=0.003,\n",
    "    save_checkpoints_steps=5000,\n",
    "    keep_checkpoint_max=3,\n",
    "    iterations_per_loop=100,\n",
    "    mesh_shape=\"model:1,batch:1\", \n",
    "    mesh_devices=[\"gpu:0\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 't5-base-summary', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "isolate_session_state: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': None, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7f3158124d68>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running train on CPU\n",
      "INFO:tensorflow:feature inputs : Tensor(\"Reshape:0\", shape=(1, 8, 1024), dtype=int32)\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/mesh_tensorflow/transformer/utils.py:386: Print (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2018-08-20.\n",
      "Instructions for updating:\n",
      "Use tf.print instead of tf.Print. Note that tf.print returns a no-output operator that directly prints the output. Outside of defuns or eager mode, this operator will not be executed unless it is directly specified in session.run or used as a control dependency for other operators. This is only a concern in graph mode. Below is an example of how to ensure tf.print executes in graph mode:\n",
      "\n",
      "INFO:tensorflow:feature inputs_position : Tensor(\"Reshape_1:0\", shape=(1, 8, 1024), dtype=int32)\n",
      "INFO:tensorflow:feature targets : Tensor(\"Reshape_2:0\", shape=(1, 8, 256), dtype=int32)\n",
      "INFO:tensorflow:feature targets_position : Tensor(\"Reshape_3:0\", shape=(1, 8, 256), dtype=int32)\n",
      "INFO:tensorflow:feature inputs_segmentation : Tensor(\"Reshape_4:0\", shape=(1, 8, 1024), dtype=int32)\n",
      "INFO:tensorflow:feature targets_segmentation : Tensor(\"Reshape_5:0\", shape=(1, 8, 256), dtype=int32)\n",
      "INFO:tensorflow:serialize_num_microbatches: tokens_per_microbatch_per_replica=8192 batch_dim=Dimension(name='batch', size=8) sequence_length={'inputs': 1024, 'targets': 256} batch_per_replica=8 num_microbatches=1\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 24674304     slice_size 24674304     Shape[vocab=32128, d_model=768]                             \n",
      "INFO:tensorflow:Trainable Variables            count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:All Variables                  count: 707     Total size: 223390336        Total slice_size: 223390336      \n",
      "INFO:tensorflow:Counters:\n",
      "allconcat: 2.05e+03\n",
      " allconcat/1: 2.05e+03\n",
      "  allconcat/1/reshape_op: 2.05e+03\n",
      "allreduce: 8.14e+08\n",
      " allreduce/[0]: 4.31e+08\n",
      "  allreduce/[0]/einsum_op: 4.31e+08\n",
      "  allreduce/[0]/reduce_op: 1.57e+05\n",
      " allreduce/[1]: 3.83e+08\n",
      "  allreduce/[1]/einsum_op: 2.23e+08\n",
      "  allreduce/[1]/reduce_op: 1.6e+08\n",
      "einsum: 4.59e+12\n",
      "einsum_unique: 4.59e+12\n",
      "output: 4.09e+10\n",
      " output/AddOperation: 8.88e+09\n",
      " output/BinaryOpWithBroadcasting: 5.4e+08\n",
      " output/BroadcastOperation: 1.93e+09\n",
      " output/Constant: 1\n",
      " output/EinsumOperation: 1.79e+10\n",
      " output/ImportOperation: 3.15e+04\n",
      " output/MinMaxOperation: 1.42e+07\n",
      " output/OneHotOperation: 8.23e+08\n",
      " output/RandomOperation: 3.64e+06\n",
      " output/RangeOperation: 1.28e+03\n",
      " output/ReduceOperation: 1.58e+09\n",
      " output/ReshapeOperation: 1.33e+09\n",
      " output/ScalarAddOperation: 2.5e+08\n",
      " output/ScalarMultiplyOperation: 4.1e+08\n",
      " output/ShiftOperation: 2.05e+03\n",
      " output/SlicewiseOperation: 5.27e+09\n",
      " output/StopGradient: 1.65e+09\n",
      " output/Variable: 2.23e+08\n",
      "output_unique: 4.09e+10\n",
      " output_unique/AddOperation: 8.88e+09\n",
      " output_unique/BinaryOpWithBroadcasting: 5.4e+08\n",
      " output_unique/BroadcastOperation: 1.93e+09\n",
      " output_unique/Constant: 1\n",
      " output_unique/EinsumOperation: 1.79e+10\n",
      " output_unique/ImportOperation: 3.15e+04\n",
      " output_unique/MinMaxOperation: 1.42e+07\n",
      " output_unique/OneHotOperation: 8.23e+08\n",
      " output_unique/RandomOperation: 3.64e+06\n",
      " output_unique/RangeOperation: 1.28e+03\n",
      " output_unique/ReduceOperation: 1.58e+09\n",
      " output_unique/ReshapeOperation: 1.33e+09\n",
      " output_unique/ScalarAddOperation: 2.5e+08\n",
      " output_unique/ScalarMultiplyOperation: 4.1e+08\n",
      " output_unique/ShiftOperation: 2.05e+03\n",
      " output_unique/SlicewiseOperation: 5.27e+09\n",
      " output_unique/StopGradient: 1.65e+09\n",
      " output_unique/Variable: 2.23e+08\n",
      "variables: 2.23e+08\n",
      " variables/trainable: 2.23e+08\n",
      " variables/untrainable: 4.87e+05\n",
      "INFO:tensorflow:Initializing variables from t5-base-bahasa/model.ckpt-110200:\n",
      "INFO:tensorflow:Variables in t5-base-bahasa/model.ckpt-110200 but not in graph:\n",
      "INFO:tensorflow:\n",
      "INFO:tensorflow:Variables in graph but not in t5-base-bahasa/model.ckpt-110200:\n",
      "INFO:tensorflow:\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting the session.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1475: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Saving checkpoints for 110200 into t5-base-summary/model.ckpt.\n",
      "INFO:tensorflow:Before Save.\n",
      "INFO:tensorflow:About to write a checkpoint\n",
      "INFO:tensorflow:Done writing checkpoint.\n",
      "INFO:tensorflow:global_step/sec: 0.0280179\n",
      "INFO:tensorflow:examples/sec: 0.224143\n",
      "INFO:tensorflow:global_step/sec: 0.748906\n",
      "INFO:tensorflow:examples/sec: 5.99125\n",
      "INFO:tensorflow:global_step/sec: 0.749481\n",
      "INFO:tensorflow:examples/sec: 5.99585\n",
      "INFO:tensorflow:global_step/sec: 0.745447\n",
      "INFO:tensorflow:examples/sec: 5.96357\n",
      "INFO:tensorflow:global_step/sec: 0.74758\n",
      "INFO:tensorflow:examples/sec: 5.98064\n",
      "INFO:tensorflow:global_step/sec: 0.752744\n",
      "INFO:tensorflow:examples/sec: 6.02195\n",
      "INFO:tensorflow:global_step/sec: 0.749088\n",
      "INFO:tensorflow:examples/sec: 5.9927\n",
      "INFO:tensorflow:global_step/sec: 0.745674\n",
      "INFO:tensorflow:examples/sec: 5.96539\n",
      "INFO:tensorflow:global_step/sec: 0.751822\n",
      "INFO:tensorflow:examples/sec: 6.01457\n",
      "INFO:tensorflow:global_step/sec: 0.744743\n",
      "INFO:tensorflow:examples/sec: 5.95794\n",
      "INFO:tensorflow:global_step/sec: 0.753363\n",
      "INFO:tensorflow:examples/sec: 6.0269\n",
      "INFO:tensorflow:global_step/sec: 0.745369\n",
      "INFO:tensorflow:examples/sec: 5.96295\n",
      "INFO:tensorflow:global_step/sec: 0.750198\n",
      "INFO:tensorflow:examples/sec: 6.00158\n",
      "INFO:tensorflow:global_step/sec: 0.753375\n",
      "INFO:tensorflow:examples/sec: 6.027\n",
      "INFO:tensorflow:global_step/sec: 0.745269\n",
      "INFO:tensorflow:examples/sec: 5.96215\n",
      "INFO:tensorflow:global_step/sec: 0.754108\n",
      "INFO:tensorflow:examples/sec: 6.03286\n",
      "INFO:tensorflow:global_step/sec: 0.746495\n",
      "INFO:tensorflow:examples/sec: 5.97196\n",
      "INFO:tensorflow:global_step/sec: 0.744881\n",
      "INFO:tensorflow:examples/sec: 5.95905\n",
      "INFO:tensorflow:global_step/sec: 0.746417\n",
      "INFO:tensorflow:examples/sec: 5.97133\n",
      "INFO:tensorflow:global_step/sec: 0.743665\n",
      "INFO:tensorflow:examples/sec: 5.94932\n",
      "INFO:tensorflow:global_step/sec: 0.742157\n",
      "INFO:tensorflow:examples/sec: 5.93726\n",
      "INFO:tensorflow:global_step/sec: 0.746471\n",
      "INFO:tensorflow:examples/sec: 5.97177\n",
      "INFO:tensorflow:global_step/sec: 0.744291\n",
      "INFO:tensorflow:examples/sec: 5.95433\n",
      "INFO:tensorflow:global_step/sec: 0.745843\n",
      "INFO:tensorflow:examples/sec: 5.96675\n",
      "INFO:tensorflow:global_step/sec: 0.744002\n",
      "INFO:tensorflow:examples/sec: 5.95202\n",
      "INFO:tensorflow:global_step/sec: 0.745192\n",
      "INFO:tensorflow:examples/sec: 5.96153\n",
      "INFO:tensorflow:global_step/sec: 0.752316\n",
      "INFO:tensorflow:examples/sec: 6.01853\n",
      "INFO:tensorflow:global_step/sec: 0.7416\n",
      "INFO:tensorflow:examples/sec: 5.9328\n",
      "INFO:tensorflow:global_step/sec: 0.744395\n",
      "INFO:tensorflow:examples/sec: 5.95516\n",
      "INFO:tensorflow:global_step/sec: 0.751163\n",
      "INFO:tensorflow:examples/sec: 6.00931\n",
      "INFO:tensorflow:global_step/sec: 0.750722\n",
      "INFO:tensorflow:examples/sec: 6.00577\n",
      "INFO:tensorflow:global_step/sec: 0.746506\n",
      "INFO:tensorflow:examples/sec: 5.97205\n",
      "INFO:tensorflow:global_step/sec: 0.748807\n",
      "INFO:tensorflow:examples/sec: 5.99045\n",
      "INFO:tensorflow:global_step/sec: 0.749913\n",
      "INFO:tensorflow:examples/sec: 5.99931\n",
      "INFO:tensorflow:global_step/sec: 0.745116\n",
      "INFO:tensorflow:examples/sec: 5.96093\n",
      "INFO:tensorflow:global_step/sec: 0.748597\n",
      "INFO:tensorflow:examples/sec: 5.98877\n",
      "INFO:tensorflow:global_step/sec: 0.748388\n",
      "INFO:tensorflow:examples/sec: 5.9871\n",
      "INFO:tensorflow:global_step/sec: 0.748618\n",
      "INFO:tensorflow:examples/sec: 5.98895\n",
      "INFO:tensorflow:global_step/sec: 0.744678\n",
      "INFO:tensorflow:examples/sec: 5.95743\n",
      "INFO:tensorflow:global_step/sec: 0.748109\n",
      "INFO:tensorflow:examples/sec: 5.98487\n",
      "INFO:tensorflow:global_step/sec: 0.745398\n",
      "INFO:tensorflow:examples/sec: 5.96319\n",
      "INFO:tensorflow:global_step/sec: 0.748429\n",
      "INFO:tensorflow:examples/sec: 5.98743\n",
      "INFO:tensorflow:global_step/sec: 0.747467\n",
      "INFO:tensorflow:examples/sec: 5.97974\n",
      "INFO:tensorflow:global_step/sec: 0.7506\n",
      "INFO:tensorflow:examples/sec: 6.0048\n",
      "INFO:tensorflow:global_step/sec: 0.744303\n",
      "INFO:tensorflow:examples/sec: 5.95442\n",
      "INFO:tensorflow:global_step/sec: 0.754575\n",
      "INFO:tensorflow:examples/sec: 6.0366\n",
      "INFO:tensorflow:global_step/sec: 0.74241\n",
      "INFO:tensorflow:examples/sec: 5.93928\n",
      "INFO:tensorflow:global_step/sec: 0.743721\n",
      "INFO:tensorflow:examples/sec: 5.94977\n",
      "INFO:tensorflow:global_step/sec: 0.746818\n",
      "INFO:tensorflow:examples/sec: 5.97455\n",
      "INFO:tensorflow:global_step/sec: 0.745531\n",
      "INFO:tensorflow:examples/sec: 5.96425\n",
      "INFO:tensorflow:global_step/sec: 0.745129\n",
      "INFO:tensorflow:examples/sec: 5.96103\n",
      "INFO:tensorflow:global_step/sec: 0.744895\n",
      "INFO:tensorflow:examples/sec: 5.95916\n",
      "INFO:tensorflow:global_step/sec: 0.748719\n",
      "INFO:tensorflow:examples/sec: 5.98975\n",
      "INFO:tensorflow:global_step/sec: 0.749013\n",
      "INFO:tensorflow:examples/sec: 5.99211\n",
      "INFO:tensorflow:global_step/sec: 0.749231\n",
      "INFO:tensorflow:examples/sec: 5.99385\n",
      "INFO:tensorflow:global_step/sec: 0.75263\n",
      "INFO:tensorflow:examples/sec: 6.02104\n",
      "INFO:tensorflow:global_step/sec: 0.746801\n",
      "INFO:tensorflow:examples/sec: 5.97441\n",
      "INFO:tensorflow:global_step/sec: 0.746008\n",
      "INFO:tensorflow:examples/sec: 5.96806\n",
      "INFO:tensorflow:global_step/sec: 0.745198\n",
      "INFO:tensorflow:examples/sec: 5.96158\n",
      "INFO:tensorflow:global_step/sec: 0.742874\n",
      "INFO:tensorflow:examples/sec: 5.94299\n",
      "INFO:tensorflow:global_step/sec: 0.745233\n",
      "INFO:tensorflow:examples/sec: 5.96186\n",
      "INFO:tensorflow:global_step/sec: 0.745232\n",
      "INFO:tensorflow:examples/sec: 5.96186\n",
      "INFO:tensorflow:global_step/sec: 0.749893\n",
      "INFO:tensorflow:examples/sec: 5.99914\n",
      "INFO:tensorflow:global_step/sec: 0.75022\n",
      "INFO:tensorflow:examples/sec: 6.00176\n",
      "INFO:tensorflow:global_step/sec: 0.744274\n",
      "INFO:tensorflow:examples/sec: 5.9542\n",
      "INFO:tensorflow:global_step/sec: 0.744111\n",
      "INFO:tensorflow:examples/sec: 5.95289\n",
      "INFO:tensorflow:global_step/sec: 0.746607\n",
      "INFO:tensorflow:examples/sec: 5.97285\n",
      "INFO:tensorflow:global_step/sec: 0.746917\n",
      "INFO:tensorflow:examples/sec: 5.97534\n",
      "INFO:tensorflow:global_step/sec: 0.747283\n",
      "INFO:tensorflow:examples/sec: 5.97827\n",
      "INFO:tensorflow:global_step/sec: 0.742965\n",
      "INFO:tensorflow:examples/sec: 5.94372\n",
      "INFO:tensorflow:global_step/sec: 0.74729\n",
      "INFO:tensorflow:examples/sec: 5.97832\n",
      "INFO:tensorflow:global_step/sec: 0.750544\n",
      "INFO:tensorflow:examples/sec: 6.00435\n",
      "INFO:tensorflow:global_step/sec: 0.74514\n",
      "INFO:tensorflow:examples/sec: 5.96112\n",
      "INFO:tensorflow:global_step/sec: 0.74739\n",
      "INFO:tensorflow:examples/sec: 5.97912\n",
      "INFO:tensorflow:global_step/sec: 0.746519\n",
      "INFO:tensorflow:examples/sec: 5.97215\n",
      "INFO:tensorflow:global_step/sec: 0.747619\n",
      "INFO:tensorflow:examples/sec: 5.98096\n",
      "INFO:tensorflow:global_step/sec: 0.74348\n",
      "INFO:tensorflow:examples/sec: 5.94784\n",
      "INFO:tensorflow:global_step/sec: 0.746276\n",
      "INFO:tensorflow:examples/sec: 5.9702\n",
      "INFO:tensorflow:global_step/sec: 0.748417\n",
      "INFO:tensorflow:examples/sec: 5.98734\n",
      "INFO:tensorflow:global_step/sec: 0.745615\n",
      "INFO:tensorflow:examples/sec: 5.96492\n",
      "INFO:tensorflow:global_step/sec: 0.745974\n",
      "INFO:tensorflow:examples/sec: 5.96779\n",
      "INFO:tensorflow:global_step/sec: 0.745429\n",
      "INFO:tensorflow:examples/sec: 5.96343\n",
      "INFO:tensorflow:global_step/sec: 0.747522\n",
      "INFO:tensorflow:examples/sec: 5.98018\n",
      "INFO:tensorflow:global_step/sec: 0.745938\n",
      "INFO:tensorflow:examples/sec: 5.9675\n",
      "INFO:tensorflow:global_step/sec: 0.742992\n",
      "INFO:tensorflow:examples/sec: 5.94394\n",
      "INFO:tensorflow:global_step/sec: 0.748687\n",
      "INFO:tensorflow:examples/sec: 5.9895\n",
      "INFO:tensorflow:global_step/sec: 0.75133\n",
      "INFO:tensorflow:examples/sec: 6.01064\n",
      "INFO:tensorflow:global_step/sec: 0.745971\n",
      "INFO:tensorflow:examples/sec: 5.96777\n",
      "INFO:tensorflow:global_step/sec: 0.744714\n",
      "INFO:tensorflow:examples/sec: 5.95771\n",
      "INFO:tensorflow:global_step/sec: 0.746993\n",
      "INFO:tensorflow:examples/sec: 5.97594\n",
      "INFO:tensorflow:global_step/sec: 0.74602\n",
      "INFO:tensorflow:examples/sec: 5.96816\n",
      "INFO:tensorflow:global_step/sec: 0.74249\n",
      "INFO:tensorflow:examples/sec: 5.93992\n",
      "INFO:tensorflow:global_step/sec: 0.750689\n",
      "INFO:tensorflow:examples/sec: 6.00551\n",
      "INFO:tensorflow:global_step/sec: 0.742952\n",
      "INFO:tensorflow:examples/sec: 5.94362\n",
      "INFO:tensorflow:global_step/sec: 0.748389\n",
      "INFO:tensorflow:examples/sec: 5.98711\n",
      "INFO:tensorflow:global_step/sec: 0.746413\n",
      "INFO:tensorflow:examples/sec: 5.9713\n",
      "INFO:tensorflow:global_step/sec: 0.746115\n",
      "INFO:tensorflow:examples/sec: 5.96892\n",
      "INFO:tensorflow:global_step/sec: 0.747496\n",
      "INFO:tensorflow:examples/sec: 5.97996\n",
      "INFO:tensorflow:global_step/sec: 0.745892\n",
      "INFO:tensorflow:examples/sec: 5.96713\n",
      "INFO:tensorflow:global_step/sec: 0.743769\n",
      "INFO:tensorflow:examples/sec: 5.95015\n",
      "INFO:tensorflow:global_step/sec: 0.745785\n",
      "INFO:tensorflow:examples/sec: 5.96628\n",
      "INFO:tensorflow:global_step/sec: 0.749605\n",
      "INFO:tensorflow:examples/sec: 5.99684\n",
      "INFO:tensorflow:global_step/sec: 0.748743\n",
      "INFO:tensorflow:examples/sec: 5.98994\n",
      "INFO:tensorflow:global_step/sec: 0.749802\n",
      "INFO:tensorflow:examples/sec: 5.99841\n",
      "INFO:tensorflow:global_step/sec: 0.74263\n",
      "INFO:tensorflow:examples/sec: 5.94104\n",
      "INFO:tensorflow:global_step/sec: 0.740501\n",
      "INFO:tensorflow:examples/sec: 5.92401\n",
      "INFO:tensorflow:global_step/sec: 0.746762\n",
      "INFO:tensorflow:examples/sec: 5.97409\n",
      "INFO:tensorflow:global_step/sec: 0.753967\n",
      "INFO:tensorflow:examples/sec: 6.03174\n",
      "INFO:tensorflow:global_step/sec: 0.748852\n",
      "INFO:tensorflow:examples/sec: 5.99082\n",
      "INFO:tensorflow:global_step/sec: 0.748082\n",
      "INFO:tensorflow:examples/sec: 5.98466\n",
      "INFO:tensorflow:global_step/sec: 0.748661\n",
      "INFO:tensorflow:examples/sec: 5.98929\n",
      "INFO:tensorflow:global_step/sec: 0.749795\n",
      "INFO:tensorflow:examples/sec: 5.99836\n",
      "INFO:tensorflow:global_step/sec: 0.743708\n",
      "INFO:tensorflow:examples/sec: 5.94966\n",
      "INFO:tensorflow:global_step/sec: 0.739503\n",
      "INFO:tensorflow:examples/sec: 5.91602\n",
      "INFO:tensorflow:global_step/sec: 0.746149\n",
      "INFO:tensorflow:examples/sec: 5.96919\n",
      "INFO:tensorflow:global_step/sec: 0.755978\n",
      "INFO:tensorflow:examples/sec: 6.04782\n",
      "INFO:tensorflow:global_step/sec: 0.748262\n",
      "INFO:tensorflow:examples/sec: 5.9861\n",
      "INFO:tensorflow:global_step/sec: 0.745084\n",
      "INFO:tensorflow:examples/sec: 5.96067\n",
      "INFO:tensorflow:global_step/sec: 0.749118\n",
      "INFO:tensorflow:examples/sec: 5.99294\n",
      "INFO:tensorflow:global_step/sec: 0.748209\n",
      "INFO:tensorflow:examples/sec: 5.98567\n",
      "INFO:tensorflow:global_step/sec: 0.744507\n",
      "INFO:tensorflow:examples/sec: 5.95605\n",
      "INFO:tensorflow:global_step/sec: 0.746947\n",
      "INFO:tensorflow:examples/sec: 5.97558\n",
      "INFO:tensorflow:global_step/sec: 0.743069\n",
      "INFO:tensorflow:examples/sec: 5.94455\n",
      "INFO:tensorflow:global_step/sec: 0.748369\n",
      "INFO:tensorflow:examples/sec: 5.98695\n",
      "INFO:tensorflow:global_step/sec: 0.744954\n",
      "INFO:tensorflow:examples/sec: 5.95963\n",
      "INFO:tensorflow:global_step/sec: 0.739833\n",
      "INFO:tensorflow:examples/sec: 5.91866\n",
      "INFO:tensorflow:global_step/sec: 0.751817\n",
      "INFO:tensorflow:examples/sec: 6.01453\n",
      "INFO:tensorflow:global_step/sec: 0.744074\n",
      "INFO:tensorflow:examples/sec: 5.95259\n",
      "INFO:tensorflow:global_step/sec: 0.746807\n",
      "INFO:tensorflow:examples/sec: 5.97446\n",
      "INFO:tensorflow:global_step/sec: 0.745217\n",
      "INFO:tensorflow:examples/sec: 5.96174\n",
      "INFO:tensorflow:global_step/sec: 0.754987\n",
      "INFO:tensorflow:examples/sec: 6.0399\n",
      "INFO:tensorflow:global_step/sec: 0.750596\n",
      "INFO:tensorflow:examples/sec: 6.00476\n",
      "INFO:tensorflow:global_step/sec: 0.749253\n",
      "INFO:tensorflow:examples/sec: 5.99403\n",
      "INFO:tensorflow:global_step/sec: 0.742873\n",
      "INFO:tensorflow:examples/sec: 5.94299\n",
      "INFO:tensorflow:global_step/sec: 0.743362\n",
      "INFO:tensorflow:examples/sec: 5.94689\n",
      "INFO:tensorflow:global_step/sec: 0.748319\n",
      "INFO:tensorflow:examples/sec: 5.98655\n",
      "INFO:tensorflow:global_step/sec: 0.748568\n",
      "INFO:tensorflow:examples/sec: 5.98855\n",
      "INFO:tensorflow:global_step/sec: 0.747133\n",
      "INFO:tensorflow:examples/sec: 5.97707\n",
      "INFO:tensorflow:global_step/sec: 0.745704\n",
      "INFO:tensorflow:examples/sec: 5.96563\n",
      "INFO:tensorflow:global_step/sec: 0.749641\n",
      "INFO:tensorflow:examples/sec: 5.99713\n",
      "INFO:tensorflow:global_step/sec: 0.744069\n",
      "INFO:tensorflow:examples/sec: 5.95255\n",
      "INFO:tensorflow:global_step/sec: 0.746076\n",
      "INFO:tensorflow:examples/sec: 5.96861\n",
      "INFO:tensorflow:global_step/sec: 0.749076\n",
      "INFO:tensorflow:examples/sec: 5.99261\n",
      "INFO:tensorflow:global_step/sec: 0.745645\n",
      "INFO:tensorflow:examples/sec: 5.96516\n",
      "INFO:tensorflow:global_step/sec: 0.74391\n",
      "INFO:tensorflow:examples/sec: 5.95128\n",
      "INFO:tensorflow:global_step/sec: 0.753521\n",
      "INFO:tensorflow:examples/sec: 6.02816\n",
      "INFO:tensorflow:global_step/sec: 0.745541\n",
      "INFO:tensorflow:examples/sec: 5.96433\n",
      "INFO:tensorflow:global_step/sec: 0.740165\n",
      "INFO:tensorflow:examples/sec: 5.92132\n",
      "INFO:tensorflow:global_step/sec: 0.751428\n",
      "INFO:tensorflow:examples/sec: 6.01143\n",
      "INFO:tensorflow:global_step/sec: 0.741311\n",
      "INFO:tensorflow:examples/sec: 5.93049\n",
      "INFO:tensorflow:global_step/sec: 0.747638\n",
      "INFO:tensorflow:examples/sec: 5.9811\n",
      "INFO:tensorflow:global_step/sec: 0.749542\n",
      "INFO:tensorflow:examples/sec: 5.99633\n",
      "INFO:tensorflow:global_step/sec: 0.744113\n",
      "INFO:tensorflow:examples/sec: 5.95291\n",
      "INFO:tensorflow:global_step/sec: 0.743081\n",
      "INFO:tensorflow:examples/sec: 5.94465\n",
      "INFO:tensorflow:global_step/sec: 0.750298\n",
      "INFO:tensorflow:examples/sec: 6.00239\n",
      "INFO:tensorflow:global_step/sec: 0.74024\n",
      "INFO:tensorflow:examples/sec: 5.92192\n",
      "INFO:tensorflow:global_step/sec: 0.747657\n",
      "INFO:tensorflow:examples/sec: 5.98126\n",
      "INFO:tensorflow:global_step/sec: 0.750947\n",
      "INFO:tensorflow:examples/sec: 6.00758\n",
      "INFO:tensorflow:global_step/sec: 0.741904\n",
      "INFO:tensorflow:examples/sec: 5.93523\n",
      "INFO:tensorflow:global_step/sec: 0.755303\n",
      "INFO:tensorflow:examples/sec: 6.04243\n",
      "INFO:tensorflow:global_step/sec: 0.743047\n",
      "INFO:tensorflow:examples/sec: 5.94437\n",
      "INFO:tensorflow:global_step/sec: 0.742476\n",
      "INFO:tensorflow:examples/sec: 5.93981\n",
      "INFO:tensorflow:global_step/sec: 0.754425\n",
      "INFO:tensorflow:examples/sec: 6.0354\n",
      "INFO:tensorflow:global_step/sec: 0.746048\n",
      "INFO:tensorflow:examples/sec: 5.96838\n",
      "INFO:tensorflow:global_step/sec: 0.744861\n",
      "INFO:tensorflow:examples/sec: 5.95888\n",
      "INFO:tensorflow:global_step/sec: 0.75258\n",
      "INFO:tensorflow:examples/sec: 6.02064\n",
      "INFO:tensorflow:global_step/sec: 0.750182\n",
      "INFO:tensorflow:examples/sec: 6.00146\n",
      "INFO:tensorflow:global_step/sec: 0.748297\n",
      "INFO:tensorflow:examples/sec: 5.98637\n",
      "INFO:tensorflow:global_step/sec: 0.744867\n",
      "INFO:tensorflow:examples/sec: 5.95894\n",
      "INFO:tensorflow:global_step/sec: 0.747998\n",
      "INFO:tensorflow:examples/sec: 5.98398\n",
      "INFO:tensorflow:global_step/sec: 0.748193\n",
      "INFO:tensorflow:examples/sec: 5.98554\n",
      "INFO:tensorflow:global_step/sec: 0.748196\n",
      "INFO:tensorflow:examples/sec: 5.98557\n",
      "INFO:tensorflow:global_step/sec: 0.745201\n",
      "INFO:tensorflow:examples/sec: 5.96161\n",
      "INFO:tensorflow:global_step/sec: 0.747451\n",
      "INFO:tensorflow:examples/sec: 5.97961\n",
      "INFO:tensorflow:global_step/sec: 0.753072\n",
      "INFO:tensorflow:examples/sec: 6.02458\n",
      "INFO:tensorflow:global_step/sec: 0.746261\n",
      "INFO:tensorflow:examples/sec: 5.97009\n",
      "INFO:tensorflow:global_step/sec: 0.747861\n",
      "INFO:tensorflow:examples/sec: 5.98289\n",
      "INFO:tensorflow:global_step/sec: 0.745671\n",
      "INFO:tensorflow:examples/sec: 5.96537\n",
      "INFO:tensorflow:global_step/sec: 0.747418\n",
      "INFO:tensorflow:examples/sec: 5.97934\n",
      "INFO:tensorflow:global_step/sec: 0.75049\n",
      "INFO:tensorflow:examples/sec: 6.00392\n",
      "INFO:tensorflow:global_step/sec: 0.749667\n",
      "INFO:tensorflow:examples/sec: 5.99734\n",
      "INFO:tensorflow:global_step/sec: 0.743422\n",
      "INFO:tensorflow:examples/sec: 5.94737\n",
      "INFO:tensorflow:global_step/sec: 0.751221\n",
      "INFO:tensorflow:examples/sec: 6.00977\n",
      "INFO:tensorflow:global_step/sec: 0.746915\n",
      "INFO:tensorflow:examples/sec: 5.97532\n",
      "INFO:tensorflow:global_step/sec: 0.746108\n",
      "INFO:tensorflow:examples/sec: 5.96887\n",
      "INFO:tensorflow:global_step/sec: 0.750902\n",
      "INFO:tensorflow:examples/sec: 6.00721\n",
      "INFO:tensorflow:global_step/sec: 0.74395\n",
      "INFO:tensorflow:examples/sec: 5.9516\n",
      "INFO:tensorflow:global_step/sec: 0.746781\n",
      "INFO:tensorflow:examples/sec: 5.97424\n",
      "INFO:tensorflow:global_step/sec: 0.745061\n",
      "INFO:tensorflow:examples/sec: 5.96049\n",
      "INFO:tensorflow:global_step/sec: 0.7464\n",
      "INFO:tensorflow:examples/sec: 5.9712\n",
      "INFO:tensorflow:global_step/sec: 0.74915\n",
      "INFO:tensorflow:examples/sec: 5.9932\n",
      "INFO:tensorflow:global_step/sec: 0.744269\n",
      "INFO:tensorflow:examples/sec: 5.95415\n",
      "INFO:tensorflow:global_step/sec: 0.749786\n",
      "INFO:tensorflow:examples/sec: 5.99829\n",
      "INFO:tensorflow:global_step/sec: 0.740435\n",
      "INFO:tensorflow:examples/sec: 5.92348\n",
      "INFO:tensorflow:global_step/sec: 0.74294\n",
      "INFO:tensorflow:examples/sec: 5.94352\n",
      "INFO:tensorflow:global_step/sec: 0.750529\n",
      "INFO:tensorflow:examples/sec: 6.00423\n",
      "INFO:tensorflow:global_step/sec: 0.743681\n",
      "INFO:tensorflow:examples/sec: 5.94945\n",
      "INFO:tensorflow:global_step/sec: 0.745837\n",
      "INFO:tensorflow:examples/sec: 5.9667\n",
      "INFO:tensorflow:global_step/sec: 0.745308\n",
      "INFO:tensorflow:examples/sec: 5.96246\n",
      "INFO:tensorflow:global_step/sec: 0.747347\n",
      "INFO:tensorflow:examples/sec: 5.97877\n",
      "INFO:tensorflow:global_step/sec: 0.739928\n",
      "INFO:tensorflow:examples/sec: 5.91943\n",
      "INFO:tensorflow:global_step/sec: 0.746602\n",
      "INFO:tensorflow:examples/sec: 5.97282\n",
      "INFO:tensorflow:global_step/sec: 0.747378\n",
      "INFO:tensorflow:examples/sec: 5.97903\n",
      "INFO:tensorflow:global_step/sec: 0.746099\n",
      "INFO:tensorflow:examples/sec: 5.96879\n",
      "INFO:tensorflow:global_step/sec: 0.74473\n",
      "INFO:tensorflow:examples/sec: 5.95784\n",
      "INFO:tensorflow:global_step/sec: 0.743111\n",
      "INFO:tensorflow:examples/sec: 5.94489\n",
      "INFO:tensorflow:global_step/sec: 0.743659\n",
      "INFO:tensorflow:examples/sec: 5.94927\n",
      "INFO:tensorflow:global_step/sec: 0.739818\n",
      "INFO:tensorflow:examples/sec: 5.91854\n",
      "INFO:tensorflow:global_step/sec: 0.74569\n",
      "INFO:tensorflow:examples/sec: 5.96552\n",
      "INFO:tensorflow:global_step/sec: 0.750483\n",
      "INFO:tensorflow:examples/sec: 6.00387\n",
      "INFO:tensorflow:global_step/sec: 0.744339\n",
      "INFO:tensorflow:examples/sec: 5.95471\n",
      "INFO:tensorflow:global_step/sec: 0.749934\n",
      "INFO:tensorflow:examples/sec: 5.99947\n",
      "INFO:tensorflow:global_step/sec: 0.743585\n",
      "INFO:tensorflow:examples/sec: 5.94868\n",
      "INFO:tensorflow:global_step/sec: 0.743145\n",
      "INFO:tensorflow:examples/sec: 5.94516\n",
      "INFO:tensorflow:global_step/sec: 0.746632\n",
      "INFO:tensorflow:examples/sec: 5.97306\n",
      "INFO:tensorflow:global_step/sec: 0.744824\n",
      "INFO:tensorflow:examples/sec: 5.95859\n",
      "INFO:tensorflow:global_step/sec: 0.748379\n",
      "INFO:tensorflow:examples/sec: 5.98703\n",
      "INFO:tensorflow:global_step/sec: 0.753881\n",
      "INFO:tensorflow:examples/sec: 6.03105\n",
      "INFO:tensorflow:global_step/sec: 0.740456\n",
      "INFO:tensorflow:examples/sec: 5.92365\n",
      "INFO:tensorflow:global_step/sec: 0.744487\n",
      "INFO:tensorflow:examples/sec: 5.9559\n",
      "INFO:tensorflow:global_step/sec: 0.746347\n",
      "INFO:tensorflow:examples/sec: 5.97078\n",
      "INFO:tensorflow:global_step/sec: 0.750759\n",
      "INFO:tensorflow:examples/sec: 6.00607\n",
      "INFO:tensorflow:global_step/sec: 0.74485\n",
      "INFO:tensorflow:examples/sec: 5.9588\n",
      "INFO:tensorflow:global_step/sec: 0.747371\n",
      "INFO:tensorflow:examples/sec: 5.97897\n",
      "INFO:tensorflow:global_step/sec: 0.747294\n",
      "INFO:tensorflow:examples/sec: 5.97835\n",
      "INFO:tensorflow:global_step/sec: 0.742503\n",
      "INFO:tensorflow:examples/sec: 5.94002\n",
      "INFO:tensorflow:global_step/sec: 0.746091\n",
      "INFO:tensorflow:examples/sec: 5.96873\n",
      "INFO:tensorflow:global_step/sec: 0.745967\n",
      "INFO:tensorflow:examples/sec: 5.96774\n",
      "INFO:tensorflow:global_step/sec: 0.746247\n",
      "INFO:tensorflow:examples/sec: 5.96998\n",
      "INFO:tensorflow:global_step/sec: 0.742612\n",
      "INFO:tensorflow:examples/sec: 5.9409\n",
      "INFO:tensorflow:global_step/sec: 0.750082\n",
      "INFO:tensorflow:examples/sec: 6.00066\n",
      "INFO:tensorflow:global_step/sec: 0.748463\n",
      "INFO:tensorflow:examples/sec: 5.9877\n",
      "INFO:tensorflow:global_step/sec: 0.746826\n",
      "INFO:tensorflow:examples/sec: 5.97461\n",
      "INFO:tensorflow:global_step/sec: 0.750221\n",
      "INFO:tensorflow:examples/sec: 6.00177\n",
      "INFO:tensorflow:global_step/sec: 0.744423\n",
      "INFO:tensorflow:examples/sec: 5.95538\n",
      "INFO:tensorflow:global_step/sec: 0.745551\n",
      "INFO:tensorflow:examples/sec: 5.96441\n",
      "INFO:tensorflow:global_step/sec: 0.751673\n",
      "INFO:tensorflow:examples/sec: 6.01338\n",
      "INFO:tensorflow:global_step/sec: 0.743605\n",
      "INFO:tensorflow:examples/sec: 5.94884\n",
      "INFO:tensorflow:global_step/sec: 0.747009\n",
      "INFO:tensorflow:examples/sec: 5.97607\n",
      "INFO:tensorflow:global_step/sec: 0.74254\n",
      "INFO:tensorflow:examples/sec: 5.94032\n",
      "INFO:tensorflow:global_step/sec: 0.741694\n",
      "INFO:tensorflow:examples/sec: 5.93355\n",
      "INFO:tensorflow:global_step/sec: 0.745345\n",
      "INFO:tensorflow:examples/sec: 5.96276\n",
      "INFO:tensorflow:global_step/sec: 0.743216\n",
      "INFO:tensorflow:examples/sec: 5.94572\n",
      "INFO:tensorflow:global_step/sec: 0.747688\n",
      "INFO:tensorflow:examples/sec: 5.9815\n",
      "INFO:tensorflow:global_step/sec: 0.743739\n",
      "INFO:tensorflow:examples/sec: 5.94991\n",
      "INFO:tensorflow:global_step/sec: 0.74612\n",
      "INFO:tensorflow:examples/sec: 5.96896\n",
      "INFO:tensorflow:global_step/sec: 0.74368\n",
      "INFO:tensorflow:examples/sec: 5.94944\n",
      "INFO:tensorflow:global_step/sec: 0.752729\n",
      "INFO:tensorflow:examples/sec: 6.02183\n",
      "INFO:tensorflow:global_step/sec: 0.750265\n",
      "INFO:tensorflow:examples/sec: 6.00212\n",
      "INFO:tensorflow:global_step/sec: 0.748324\n",
      "INFO:tensorflow:examples/sec: 5.98659\n",
      "INFO:tensorflow:global_step/sec: 0.739299\n",
      "INFO:tensorflow:examples/sec: 5.91439\n",
      "INFO:tensorflow:global_step/sec: 0.74852\n",
      "INFO:tensorflow:examples/sec: 5.98816\n",
      "INFO:tensorflow:global_step/sec: 0.740749\n",
      "INFO:tensorflow:examples/sec: 5.92599\n",
      "INFO:tensorflow:global_step/sec: 0.747853\n",
      "INFO:tensorflow:examples/sec: 5.98283\n",
      "INFO:tensorflow:global_step/sec: 0.746992\n",
      "INFO:tensorflow:examples/sec: 5.97593\n",
      "INFO:tensorflow:global_step/sec: 0.744526\n",
      "INFO:tensorflow:examples/sec: 5.95621\n",
      "INFO:tensorflow:global_step/sec: 0.741324\n",
      "INFO:tensorflow:examples/sec: 5.93059\n",
      "INFO:tensorflow:global_step/sec: 0.742582\n",
      "INFO:tensorflow:examples/sec: 5.94066\n",
      "INFO:tensorflow:global_step/sec: 0.746171\n",
      "INFO:tensorflow:examples/sec: 5.96937\n",
      "INFO:tensorflow:global_step/sec: 0.747719\n",
      "INFO:tensorflow:examples/sec: 5.98175\n",
      "INFO:tensorflow:global_step/sec: 0.744963\n",
      "INFO:tensorflow:examples/sec: 5.9597\n",
      "INFO:tensorflow:global_step/sec: 0.745484\n",
      "INFO:tensorflow:examples/sec: 5.96387\n",
      "INFO:tensorflow:global_step/sec: 0.740058\n",
      "INFO:tensorflow:examples/sec: 5.92047\n",
      "INFO:tensorflow:global_step/sec: 0.746223\n",
      "INFO:tensorflow:examples/sec: 5.96978\n",
      "INFO:tensorflow:global_step/sec: 0.743092\n",
      "INFO:tensorflow:examples/sec: 5.94474\n",
      "INFO:tensorflow:global_step/sec: 0.743196\n",
      "INFO:tensorflow:examples/sec: 5.94557\n",
      "INFO:tensorflow:global_step/sec: 0.741433\n",
      "INFO:tensorflow:examples/sec: 5.93146\n",
      "INFO:tensorflow:global_step/sec: 0.739253\n",
      "INFO:tensorflow:examples/sec: 5.91403\n",
      "INFO:tensorflow:global_step/sec: 0.743887\n",
      "INFO:tensorflow:examples/sec: 5.9511\n",
      "INFO:tensorflow:global_step/sec: 0.747484\n",
      "INFO:tensorflow:examples/sec: 5.97987\n",
      "INFO:tensorflow:global_step/sec: 0.745625\n",
      "INFO:tensorflow:examples/sec: 5.965\n",
      "INFO:tensorflow:global_step/sec: 0.744034\n",
      "INFO:tensorflow:examples/sec: 5.95227\n",
      "INFO:tensorflow:global_step/sec: 0.743493\n",
      "INFO:tensorflow:examples/sec: 5.94794\n",
      "INFO:tensorflow:global_step/sec: 0.751626\n",
      "INFO:tensorflow:examples/sec: 6.01301\n",
      "INFO:tensorflow:global_step/sec: 0.741859\n",
      "INFO:tensorflow:examples/sec: 5.93487\n",
      "INFO:tensorflow:global_step/sec: 0.752566\n",
      "INFO:tensorflow:examples/sec: 6.02053\n",
      "INFO:tensorflow:global_step/sec: 0.745595\n",
      "INFO:tensorflow:examples/sec: 5.96476\n",
      "INFO:tensorflow:global_step/sec: 0.741181\n",
      "INFO:tensorflow:examples/sec: 5.92944\n",
      "INFO:tensorflow:global_step/sec: 0.744896\n",
      "INFO:tensorflow:examples/sec: 5.95917\n",
      "INFO:tensorflow:global_step/sec: 0.744799\n",
      "INFO:tensorflow:examples/sec: 5.95839\n",
      "INFO:tensorflow:global_step/sec: 0.749379\n",
      "INFO:tensorflow:examples/sec: 5.99503\n",
      "INFO:tensorflow:global_step/sec: 0.746752\n",
      "INFO:tensorflow:examples/sec: 5.97401\n",
      "INFO:tensorflow:global_step/sec: 0.740237\n",
      "INFO:tensorflow:examples/sec: 5.9219\n",
      "INFO:tensorflow:global_step/sec: 0.74401\n",
      "INFO:tensorflow:examples/sec: 5.95208\n",
      "INFO:tensorflow:global_step/sec: 0.743604\n",
      "INFO:tensorflow:examples/sec: 5.94883\n",
      "INFO:tensorflow:global_step/sec: 0.74687\n",
      "INFO:tensorflow:examples/sec: 5.97496\n",
      "INFO:tensorflow:global_step/sec: 0.744524\n",
      "INFO:tensorflow:examples/sec: 5.9562\n",
      "INFO:tensorflow:global_step/sec: 0.745754\n",
      "INFO:tensorflow:examples/sec: 5.96603\n",
      "INFO:tensorflow:global_step/sec: 0.744416\n",
      "INFO:tensorflow:examples/sec: 5.95533\n",
      "INFO:tensorflow:global_step/sec: 0.751746\n",
      "INFO:tensorflow:examples/sec: 6.01397\n",
      "INFO:tensorflow:global_step/sec: 0.747076\n",
      "INFO:tensorflow:examples/sec: 5.97661\n",
      "INFO:tensorflow:global_step/sec: 0.743328\n",
      "INFO:tensorflow:examples/sec: 5.94663\n",
      "INFO:tensorflow:global_step/sec: 0.746772\n",
      "INFO:tensorflow:examples/sec: 5.97418\n",
      "INFO:tensorflow:global_step/sec: 0.745336\n",
      "INFO:tensorflow:examples/sec: 5.96269\n",
      "INFO:tensorflow:global_step/sec: 0.743894\n",
      "INFO:tensorflow:examples/sec: 5.95115\n",
      "INFO:tensorflow:global_step/sec: 0.74324\n",
      "INFO:tensorflow:examples/sec: 5.94592\n",
      "INFO:tensorflow:global_step/sec: 0.745571\n",
      "INFO:tensorflow:examples/sec: 5.96457\n",
      "INFO:tensorflow:global_step/sec: 0.740309\n",
      "INFO:tensorflow:examples/sec: 5.92248\n",
      "INFO:tensorflow:global_step/sec: 0.753795\n",
      "INFO:tensorflow:examples/sec: 6.03036\n",
      "INFO:tensorflow:global_step/sec: 0.743465\n",
      "INFO:tensorflow:examples/sec: 5.94772\n",
      "INFO:tensorflow:global_step/sec: 0.746774\n",
      "INFO:tensorflow:examples/sec: 5.97419\n",
      "INFO:tensorflow:global_step/sec: 0.747782\n",
      "INFO:tensorflow:examples/sec: 5.98226\n",
      "INFO:tensorflow:global_step/sec: 0.746958\n",
      "INFO:tensorflow:examples/sec: 5.97566\n",
      "INFO:tensorflow:global_step/sec: 0.746805\n",
      "INFO:tensorflow:examples/sec: 5.97444\n",
      "INFO:tensorflow:global_step/sec: 0.753504\n",
      "INFO:tensorflow:examples/sec: 6.02803\n",
      "INFO:tensorflow:global_step/sec: 0.74372\n",
      "INFO:tensorflow:examples/sec: 5.94976\n",
      "INFO:tensorflow:global_step/sec: 0.744944\n",
      "INFO:tensorflow:examples/sec: 5.95955\n",
      "INFO:tensorflow:global_step/sec: 0.754192\n",
      "INFO:tensorflow:examples/sec: 6.03354\n",
      "INFO:tensorflow:global_step/sec: 0.748441\n",
      "INFO:tensorflow:examples/sec: 5.98752\n",
      "INFO:tensorflow:global_step/sec: 0.738622\n",
      "INFO:tensorflow:examples/sec: 5.90898\n",
      "INFO:tensorflow:global_step/sec: 0.742997\n",
      "INFO:tensorflow:examples/sec: 5.94398\n",
      "INFO:tensorflow:global_step/sec: 0.743844\n",
      "INFO:tensorflow:examples/sec: 5.95075\n",
      "INFO:tensorflow:global_step/sec: 0.743343\n",
      "INFO:tensorflow:examples/sec: 5.94674\n",
      "INFO:tensorflow:global_step/sec: 0.747735\n",
      "INFO:tensorflow:examples/sec: 5.98188\n",
      "INFO:tensorflow:global_step/sec: 0.751246\n",
      "INFO:tensorflow:examples/sec: 6.00997\n",
      "INFO:tensorflow:global_step/sec: 0.753448\n",
      "INFO:tensorflow:examples/sec: 6.02758\n",
      "INFO:tensorflow:global_step/sec: 0.74445\n",
      "INFO:tensorflow:examples/sec: 5.9556\n",
      "INFO:tensorflow:global_step/sec: 0.741444\n",
      "INFO:tensorflow:examples/sec: 5.93155\n",
      "INFO:tensorflow:global_step/sec: 0.744368\n",
      "INFO:tensorflow:examples/sec: 5.95495\n",
      "INFO:tensorflow:global_step/sec: 0.746549\n",
      "INFO:tensorflow:examples/sec: 5.97239\n",
      "INFO:tensorflow:global_step/sec: 0.750536\n",
      "INFO:tensorflow:examples/sec: 6.00429\n",
      "INFO:tensorflow:global_step/sec: 0.743006\n",
      "INFO:tensorflow:examples/sec: 5.94405\n",
      "INFO:tensorflow:global_step/sec: 0.747533\n",
      "INFO:tensorflow:examples/sec: 5.98026\n",
      "INFO:tensorflow:global_step/sec: 0.740384\n",
      "INFO:tensorflow:examples/sec: 5.92307\n",
      "INFO:tensorflow:global_step/sec: 0.748524\n",
      "INFO:tensorflow:examples/sec: 5.98819\n",
      "INFO:tensorflow:global_step/sec: 0.750943\n",
      "INFO:tensorflow:examples/sec: 6.00754\n",
      "INFO:tensorflow:global_step/sec: 0.745602\n",
      "INFO:tensorflow:examples/sec: 5.96482\n",
      "INFO:tensorflow:global_step/sec: 0.745937\n",
      "INFO:tensorflow:examples/sec: 5.96749\n",
      "INFO:tensorflow:global_step/sec: 0.744329\n",
      "INFO:tensorflow:examples/sec: 5.95463\n",
      "INFO:tensorflow:global_step/sec: 0.742761\n",
      "INFO:tensorflow:examples/sec: 5.94209\n",
      "INFO:tensorflow:global_step/sec: 0.742036\n",
      "INFO:tensorflow:examples/sec: 5.93629\n",
      "INFO:tensorflow:global_step/sec: 0.746541\n",
      "INFO:tensorflow:examples/sec: 5.97233\n",
      "INFO:tensorflow:global_step/sec: 0.75019\n",
      "INFO:tensorflow:examples/sec: 6.00152\n",
      "INFO:tensorflow:global_step/sec: 0.746364\n",
      "INFO:tensorflow:examples/sec: 5.97091\n",
      "INFO:tensorflow:global_step/sec: 0.745635\n",
      "INFO:tensorflow:examples/sec: 5.96508\n",
      "INFO:tensorflow:global_step/sec: 0.74458\n",
      "INFO:tensorflow:examples/sec: 5.95664\n",
      "INFO:tensorflow:global_step/sec: 0.743255\n",
      "INFO:tensorflow:examples/sec: 5.94604\n",
      "INFO:tensorflow:global_step/sec: 0.746021\n",
      "INFO:tensorflow:examples/sec: 5.96817\n",
      "INFO:tensorflow:global_step/sec: 0.744342\n",
      "INFO:tensorflow:examples/sec: 5.95474\n",
      "INFO:tensorflow:global_step/sec: 0.749228\n",
      "INFO:tensorflow:examples/sec: 5.99382\n",
      "INFO:tensorflow:global_step/sec: 0.747913\n",
      "INFO:tensorflow:examples/sec: 5.98331\n",
      "INFO:tensorflow:global_step/sec: 0.742669\n",
      "INFO:tensorflow:examples/sec: 5.94135\n",
      "INFO:tensorflow:global_step/sec: 0.744189\n",
      "INFO:tensorflow:examples/sec: 5.95351\n",
      "INFO:tensorflow:global_step/sec: 0.748572\n",
      "INFO:tensorflow:examples/sec: 5.98858\n",
      "INFO:tensorflow:global_step/sec: 0.749051\n",
      "INFO:tensorflow:examples/sec: 5.9924\n",
      "INFO:tensorflow:global_step/sec: 0.748554\n",
      "INFO:tensorflow:examples/sec: 5.98843\n",
      "INFO:tensorflow:global_step/sec: 0.747749\n",
      "INFO:tensorflow:examples/sec: 5.98199\n",
      "INFO:tensorflow:global_step/sec: 0.747201\n",
      "INFO:tensorflow:examples/sec: 5.9776\n",
      "INFO:tensorflow:global_step/sec: 0.746649\n",
      "INFO:tensorflow:examples/sec: 5.97319\n",
      "INFO:tensorflow:global_step/sec: 0.744304\n",
      "INFO:tensorflow:examples/sec: 5.95443\n",
      "INFO:tensorflow:global_step/sec: 0.741405\n",
      "INFO:tensorflow:examples/sec: 5.93124\n",
      "INFO:tensorflow:global_step/sec: 0.739258\n",
      "INFO:tensorflow:examples/sec: 5.91407\n",
      "INFO:tensorflow:global_step/sec: 0.742511\n",
      "INFO:tensorflow:examples/sec: 5.94009\n",
      "INFO:tensorflow:global_step/sec: 0.751422\n",
      "INFO:tensorflow:examples/sec: 6.01138\n",
      "INFO:tensorflow:global_step/sec: 0.741785\n",
      "INFO:tensorflow:examples/sec: 5.93428\n",
      "INFO:tensorflow:global_step/sec: 0.750853\n",
      "INFO:tensorflow:examples/sec: 6.00683\n",
      "INFO:tensorflow:global_step/sec: 0.740204\n",
      "INFO:tensorflow:examples/sec: 5.92163\n",
      "INFO:tensorflow:global_step/sec: 0.744102\n",
      "INFO:tensorflow:examples/sec: 5.95281\n",
      "INFO:tensorflow:global_step/sec: 0.745111\n",
      "INFO:tensorflow:examples/sec: 5.96089\n",
      "INFO:tensorflow:global_step/sec: 0.740824\n",
      "INFO:tensorflow:examples/sec: 5.92659\n",
      "INFO:tensorflow:global_step/sec: 0.745738\n",
      "INFO:tensorflow:examples/sec: 5.96591\n",
      "INFO:tensorflow:global_step/sec: 0.743343\n",
      "INFO:tensorflow:examples/sec: 5.94674\n",
      "INFO:tensorflow:global_step/sec: 0.748641\n",
      "INFO:tensorflow:examples/sec: 5.98913\n",
      "INFO:tensorflow:global_step/sec: 0.74121\n",
      "INFO:tensorflow:examples/sec: 5.92968\n",
      "INFO:tensorflow:global_step/sec: 0.750683\n",
      "INFO:tensorflow:examples/sec: 6.00546\n",
      "INFO:tensorflow:global_step/sec: 0.750648\n",
      "INFO:tensorflow:examples/sec: 6.00518\n",
      "INFO:tensorflow:global_step/sec: 0.744079\n",
      "INFO:tensorflow:examples/sec: 5.95263\n",
      "INFO:tensorflow:global_step/sec: 0.742156\n",
      "INFO:tensorflow:examples/sec: 5.93725\n",
      "INFO:tensorflow:global_step/sec: 0.742895\n",
      "INFO:tensorflow:examples/sec: 5.94316\n",
      "INFO:tensorflow:global_step/sec: 0.748192\n",
      "INFO:tensorflow:examples/sec: 5.98554\n",
      "INFO:tensorflow:global_step/sec: 0.746487\n",
      "INFO:tensorflow:examples/sec: 5.97189\n",
      "INFO:tensorflow:global_step/sec: 0.755738\n",
      "INFO:tensorflow:examples/sec: 6.0459\n",
      "INFO:tensorflow:global_step/sec: 0.751576\n",
      "INFO:tensorflow:examples/sec: 6.01261\n",
      "INFO:tensorflow:global_step/sec: 0.741339\n",
      "INFO:tensorflow:examples/sec: 5.93071\n",
      "INFO:tensorflow:global_step/sec: 0.738698\n",
      "INFO:tensorflow:examples/sec: 5.90958\n",
      "INFO:tensorflow:global_step/sec: 0.745647\n",
      "INFO:tensorflow:examples/sec: 5.96518\n",
      "INFO:tensorflow:global_step/sec: 0.742691\n",
      "INFO:tensorflow:examples/sec: 5.94152\n",
      "INFO:tensorflow:global_step/sec: 0.743565\n",
      "INFO:tensorflow:examples/sec: 5.94852\n",
      "INFO:tensorflow:global_step/sec: 0.746678\n",
      "INFO:tensorflow:examples/sec: 5.97342\n",
      "INFO:tensorflow:global_step/sec: 0.744999\n",
      "INFO:tensorflow:examples/sec: 5.95999\n",
      "INFO:tensorflow:global_step/sec: 0.750745\n",
      "INFO:tensorflow:examples/sec: 6.00596\n",
      "INFO:tensorflow:global_step/sec: 0.743911\n",
      "INFO:tensorflow:examples/sec: 5.95129\n",
      "INFO:tensorflow:global_step/sec: 0.749546\n",
      "INFO:tensorflow:examples/sec: 5.99637\n",
      "INFO:tensorflow:global_step/sec: 0.749583\n",
      "INFO:tensorflow:examples/sec: 5.99666\n",
      "INFO:tensorflow:global_step/sec: 0.749417\n",
      "INFO:tensorflow:examples/sec: 5.99534\n",
      "INFO:tensorflow:global_step/sec: 0.744949\n",
      "INFO:tensorflow:examples/sec: 5.95959\n",
      "INFO:tensorflow:global_step/sec: 0.75056\n",
      "INFO:tensorflow:examples/sec: 6.00448\n",
      "INFO:tensorflow:global_step/sec: 0.745469\n",
      "INFO:tensorflow:examples/sec: 5.96375\n",
      "INFO:tensorflow:global_step/sec: 0.743142\n",
      "INFO:tensorflow:examples/sec: 5.94514\n",
      "INFO:tensorflow:global_step/sec: 0.744117\n",
      "INFO:tensorflow:examples/sec: 5.95293\n",
      "INFO:tensorflow:global_step/sec: 0.740379\n",
      "INFO:tensorflow:examples/sec: 5.92303\n",
      "INFO:tensorflow:global_step/sec: 0.74762\n",
      "INFO:tensorflow:examples/sec: 5.98096\n",
      "INFO:tensorflow:global_step/sec: 0.747609\n",
      "INFO:tensorflow:examples/sec: 5.98087\n",
      "INFO:tensorflow:global_step/sec: 0.747624\n",
      "INFO:tensorflow:examples/sec: 5.98099\n",
      "INFO:tensorflow:global_step/sec: 0.747412\n",
      "INFO:tensorflow:examples/sec: 5.9793\n",
      "INFO:tensorflow:global_step/sec: 0.750158\n",
      "INFO:tensorflow:examples/sec: 6.00126\n",
      "INFO:tensorflow:global_step/sec: 0.74256\n",
      "INFO:tensorflow:examples/sec: 5.94048\n",
      "INFO:tensorflow:global_step/sec: 0.750879\n",
      "INFO:tensorflow:examples/sec: 6.00703\n",
      "INFO:tensorflow:global_step/sec: 0.746645\n",
      "INFO:tensorflow:examples/sec: 5.97316\n",
      "INFO:tensorflow:global_step/sec: 0.74629\n",
      "INFO:tensorflow:examples/sec: 5.97032\n",
      "INFO:tensorflow:global_step/sec: 0.754012\n",
      "INFO:tensorflow:examples/sec: 6.0321\n",
      "INFO:tensorflow:global_step/sec: 0.745634\n",
      "INFO:tensorflow:examples/sec: 5.96507\n",
      "INFO:tensorflow:global_step/sec: 0.746069\n",
      "INFO:tensorflow:examples/sec: 5.96855\n",
      "INFO:tensorflow:global_step/sec: 0.743076\n",
      "INFO:tensorflow:examples/sec: 5.94461\n",
      "INFO:tensorflow:global_step/sec: 0.748016\n",
      "INFO:tensorflow:examples/sec: 5.98413\n",
      "INFO:tensorflow:global_step/sec: 0.745514\n",
      "INFO:tensorflow:examples/sec: 5.96411\n",
      "INFO:tensorflow:global_step/sec: 0.742561\n",
      "INFO:tensorflow:examples/sec: 5.94049\n",
      "INFO:tensorflow:global_step/sec: 0.742653\n",
      "INFO:tensorflow:examples/sec: 5.94122\n",
      "INFO:tensorflow:global_step/sec: 0.747935\n",
      "INFO:tensorflow:examples/sec: 5.98348\n",
      "INFO:tensorflow:global_step/sec: 0.742826\n",
      "INFO:tensorflow:examples/sec: 5.94261\n",
      "INFO:tensorflow:global_step/sec: 0.745207\n",
      "INFO:tensorflow:examples/sec: 5.96165\n",
      "INFO:tensorflow:global_step/sec: 0.74583\n",
      "INFO:tensorflow:examples/sec: 5.96664\n",
      "INFO:tensorflow:global_step/sec: 0.748624\n",
      "INFO:tensorflow:examples/sec: 5.98899\n",
      "INFO:tensorflow:global_step/sec: 0.748236\n",
      "INFO:tensorflow:examples/sec: 5.98589\n",
      "INFO:tensorflow:global_step/sec: 0.74714\n",
      "INFO:tensorflow:examples/sec: 5.97712\n",
      "INFO:tensorflow:global_step/sec: 0.744897\n",
      "INFO:tensorflow:examples/sec: 5.95918\n",
      "INFO:tensorflow:global_step/sec: 0.741285\n",
      "INFO:tensorflow:examples/sec: 5.93028\n",
      "INFO:tensorflow:global_step/sec: 0.746753\n",
      "INFO:tensorflow:examples/sec: 5.97403\n",
      "INFO:tensorflow:global_step/sec: 0.744628\n",
      "INFO:tensorflow:examples/sec: 5.95702\n",
      "INFO:tensorflow:global_step/sec: 0.748402\n",
      "INFO:tensorflow:examples/sec: 5.98722\n",
      "INFO:tensorflow:global_step/sec: 0.74995\n",
      "INFO:tensorflow:examples/sec: 5.9996\n",
      "INFO:tensorflow:global_step/sec: 0.744366\n",
      "INFO:tensorflow:examples/sec: 5.95493\n",
      "INFO:tensorflow:global_step/sec: 0.744812\n",
      "INFO:tensorflow:examples/sec: 5.9585\n",
      "INFO:tensorflow:global_step/sec: 0.743601\n",
      "INFO:tensorflow:examples/sec: 5.94881\n",
      "INFO:tensorflow:global_step/sec: 0.74783\n",
      "INFO:tensorflow:examples/sec: 5.98264\n",
      "INFO:tensorflow:global_step/sec: 0.74641\n",
      "INFO:tensorflow:examples/sec: 5.97128\n",
      "INFO:tensorflow:global_step/sec: 0.748689\n",
      "INFO:tensorflow:examples/sec: 5.98951\n",
      "INFO:tensorflow:global_step/sec: 0.746318\n",
      "INFO:tensorflow:examples/sec: 5.97054\n",
      "INFO:tensorflow:global_step/sec: 0.74706\n",
      "INFO:tensorflow:examples/sec: 5.97648\n",
      "INFO:tensorflow:global_step/sec: 0.746761\n",
      "INFO:tensorflow:examples/sec: 5.97409\n",
      "INFO:tensorflow:global_step/sec: 0.746101\n",
      "INFO:tensorflow:examples/sec: 5.96881\n",
      "INFO:tensorflow:global_step/sec: 0.743195\n",
      "INFO:tensorflow:examples/sec: 5.94556\n",
      "INFO:tensorflow:global_step/sec: 0.748467\n",
      "INFO:tensorflow:examples/sec: 5.98774\n",
      "INFO:tensorflow:global_step/sec: 0.746134\n",
      "INFO:tensorflow:examples/sec: 5.96908\n",
      "INFO:tensorflow:global_step/sec: 0.743081\n",
      "INFO:tensorflow:examples/sec: 5.94465\n",
      "INFO:tensorflow:global_step/sec: 0.739886\n",
      "INFO:tensorflow:examples/sec: 5.91909\n",
      "INFO:tensorflow:global_step/sec: 0.75118\n",
      "INFO:tensorflow:examples/sec: 6.00944\n",
      "INFO:tensorflow:global_step/sec: 0.745435\n",
      "INFO:tensorflow:examples/sec: 5.96348\n",
      "INFO:tensorflow:global_step/sec: 0.746596\n",
      "INFO:tensorflow:examples/sec: 5.97277\n",
      "INFO:tensorflow:global_step/sec: 0.74336\n",
      "INFO:tensorflow:examples/sec: 5.94688\n",
      "INFO:tensorflow:global_step/sec: 0.736331\n",
      "INFO:tensorflow:examples/sec: 5.89065\n",
      "INFO:tensorflow:global_step/sec: 0.75311\n",
      "INFO:tensorflow:examples/sec: 6.02488\n",
      "INFO:tensorflow:global_step/sec: 0.744037\n",
      "INFO:tensorflow:examples/sec: 5.9523\n",
      "INFO:tensorflow:global_step/sec: 0.749076\n",
      "INFO:tensorflow:examples/sec: 5.99261\n",
      "INFO:tensorflow:global_step/sec: 0.744999\n",
      "INFO:tensorflow:examples/sec: 5.95999\n",
      "INFO:tensorflow:global_step/sec: 0.747925\n",
      "INFO:tensorflow:examples/sec: 5.9834\n",
      "INFO:tensorflow:global_step/sec: 0.741187\n",
      "INFO:tensorflow:examples/sec: 5.9295\n",
      "INFO:tensorflow:global_step/sec: 0.743981\n",
      "INFO:tensorflow:examples/sec: 5.95184\n",
      "INFO:tensorflow:global_step/sec: 0.744655\n",
      "INFO:tensorflow:examples/sec: 5.95724\n",
      "INFO:tensorflow:global_step/sec: 0.745112\n",
      "INFO:tensorflow:examples/sec: 5.9609\n",
      "INFO:tensorflow:global_step/sec: 0.746179\n",
      "INFO:tensorflow:examples/sec: 5.96943\n",
      "INFO:tensorflow:global_step/sec: 0.745948\n",
      "INFO:tensorflow:examples/sec: 5.96758\n",
      "INFO:tensorflow:global_step/sec: 0.748441\n",
      "INFO:tensorflow:examples/sec: 5.98753\n",
      "INFO:tensorflow:global_step/sec: 0.748169\n",
      "INFO:tensorflow:examples/sec: 5.98535\n",
      "INFO:tensorflow:global_step/sec: 0.743627\n",
      "INFO:tensorflow:examples/sec: 5.94902\n",
      "INFO:tensorflow:global_step/sec: 0.742908\n",
      "INFO:tensorflow:examples/sec: 5.94326\n",
      "INFO:tensorflow:global_step/sec: 0.747273\n",
      "INFO:tensorflow:examples/sec: 5.97818\n",
      "INFO:tensorflow:global_step/sec: 0.745906\n",
      "INFO:tensorflow:examples/sec: 5.96725\n",
      "INFO:tensorflow:global_step/sec: 0.744611\n",
      "INFO:tensorflow:examples/sec: 5.95689\n",
      "INFO:tensorflow:global_step/sec: 0.750161\n",
      "INFO:tensorflow:examples/sec: 6.00129\n",
      "INFO:tensorflow:global_step/sec: 0.743977\n",
      "INFO:tensorflow:examples/sec: 5.95182\n",
      "INFO:tensorflow:global_step/sec: 0.742815\n",
      "INFO:tensorflow:examples/sec: 5.94252\n",
      "INFO:tensorflow:global_step/sec: 0.746494\n",
      "INFO:tensorflow:examples/sec: 5.97195\n",
      "INFO:tensorflow:global_step/sec: 0.745572\n",
      "INFO:tensorflow:examples/sec: 5.96457\n",
      "INFO:tensorflow:global_step/sec: 0.740252\n",
      "INFO:tensorflow:examples/sec: 5.92202\n",
      "INFO:tensorflow:global_step/sec: 0.745929\n",
      "INFO:tensorflow:examples/sec: 5.96743\n",
      "INFO:tensorflow:global_step/sec: 0.74387\n",
      "INFO:tensorflow:examples/sec: 5.95096\n",
      "INFO:tensorflow:global_step/sec: 0.748387\n",
      "INFO:tensorflow:examples/sec: 5.98709\n",
      "INFO:tensorflow:global_step/sec: 0.740306\n",
      "INFO:tensorflow:examples/sec: 5.92245\n",
      "INFO:tensorflow:global_step/sec: 0.743346\n",
      "INFO:tensorflow:examples/sec: 5.94677\n",
      "INFO:tensorflow:global_step/sec: 0.749507\n",
      "INFO:tensorflow:examples/sec: 5.99605\n",
      "INFO:tensorflow:global_step/sec: 0.744133\n",
      "INFO:tensorflow:examples/sec: 5.95307\n",
      "INFO:tensorflow:global_step/sec: 0.747691\n",
      "INFO:tensorflow:examples/sec: 5.98153\n",
      "INFO:tensorflow:global_step/sec: 0.742249\n",
      "INFO:tensorflow:examples/sec: 5.93799\n",
      "INFO:tensorflow:global_step/sec: 0.747924\n",
      "INFO:tensorflow:examples/sec: 5.98339\n",
      "INFO:tensorflow:global_step/sec: 0.742821\n",
      "INFO:tensorflow:examples/sec: 5.94257\n",
      "INFO:tensorflow:global_step/sec: 0.744842\n",
      "INFO:tensorflow:examples/sec: 5.95874\n",
      "INFO:tensorflow:global_step/sec: 0.752271\n",
      "INFO:tensorflow:examples/sec: 6.01817\n",
      "INFO:tensorflow:global_step/sec: 0.744433\n",
      "INFO:tensorflow:examples/sec: 5.95546\n",
      "INFO:tensorflow:global_step/sec: 0.741796\n",
      "INFO:tensorflow:examples/sec: 5.93437\n",
      "INFO:tensorflow:global_step/sec: 0.748041\n",
      "INFO:tensorflow:examples/sec: 5.98433\n",
      "INFO:tensorflow:global_step/sec: 0.74759\n",
      "INFO:tensorflow:examples/sec: 5.98072\n",
      "INFO:tensorflow:global_step/sec: 0.74326\n",
      "INFO:tensorflow:examples/sec: 5.94608\n",
      "INFO:tensorflow:global_step/sec: 0.745222\n",
      "INFO:tensorflow:examples/sec: 5.96178\n",
      "INFO:tensorflow:global_step/sec: 0.746706\n",
      "INFO:tensorflow:examples/sec: 5.97365\n",
      "INFO:tensorflow:global_step/sec: 0.745434\n",
      "INFO:tensorflow:examples/sec: 5.96347\n",
      "INFO:tensorflow:global_step/sec: 0.747316\n",
      "INFO:tensorflow:examples/sec: 5.97853\n",
      "INFO:tensorflow:global_step/sec: 0.741163\n",
      "INFO:tensorflow:examples/sec: 5.92931\n",
      "INFO:tensorflow:global_step/sec: 0.743846\n",
      "INFO:tensorflow:examples/sec: 5.95077\n",
      "INFO:tensorflow:global_step/sec: 0.745801\n",
      "INFO:tensorflow:examples/sec: 5.96641\n",
      "INFO:tensorflow:global_step/sec: 0.74171\n",
      "INFO:tensorflow:examples/sec: 5.93368\n",
      "INFO:tensorflow:global_step/sec: 0.753428\n",
      "INFO:tensorflow:examples/sec: 6.02742\n",
      "INFO:tensorflow:global_step/sec: 0.742119\n",
      "INFO:tensorflow:examples/sec: 5.93695\n",
      "INFO:tensorflow:global_step/sec: 0.742276\n",
      "INFO:tensorflow:examples/sec: 5.93821\n",
      "INFO:tensorflow:global_step/sec: 0.744901\n",
      "INFO:tensorflow:examples/sec: 5.95921\n",
      "INFO:tensorflow:global_step/sec: 0.745393\n",
      "INFO:tensorflow:examples/sec: 5.96314\n",
      "INFO:tensorflow:global_step/sec: 0.741496\n",
      "INFO:tensorflow:examples/sec: 5.93197\n",
      "INFO:tensorflow:global_step/sec: 0.750001\n",
      "INFO:tensorflow:examples/sec: 6.00001\n",
      "INFO:tensorflow:global_step/sec: 0.746102\n",
      "INFO:tensorflow:examples/sec: 5.96882\n",
      "INFO:tensorflow:global_step/sec: 0.743832\n",
      "INFO:tensorflow:examples/sec: 5.95066\n",
      "INFO:tensorflow:global_step/sec: 0.745659\n",
      "INFO:tensorflow:examples/sec: 5.96527\n",
      "INFO:tensorflow:global_step/sec: 0.74262\n",
      "INFO:tensorflow:examples/sec: 5.94096\n",
      "INFO:tensorflow:global_step/sec: 0.744234\n",
      "INFO:tensorflow:examples/sec: 5.95387\n",
      "INFO:tensorflow:global_step/sec: 0.744763\n",
      "INFO:tensorflow:examples/sec: 5.95811\n",
      "INFO:tensorflow:global_step/sec: 0.743551\n",
      "INFO:tensorflow:examples/sec: 5.94841\n",
      "INFO:tensorflow:global_step/sec: 0.749313\n",
      "INFO:tensorflow:examples/sec: 5.9945\n",
      "INFO:tensorflow:global_step/sec: 0.743518\n",
      "INFO:tensorflow:examples/sec: 5.94814\n",
      "INFO:tensorflow:global_step/sec: 0.737399\n",
      "INFO:tensorflow:examples/sec: 5.89919\n",
      "INFO:tensorflow:global_step/sec: 0.745009\n",
      "INFO:tensorflow:examples/sec: 5.96007\n",
      "INFO:tensorflow:global_step/sec: 0.7451\n",
      "INFO:tensorflow:examples/sec: 5.9608\n",
      "INFO:tensorflow:global_step/sec: 0.741078\n",
      "INFO:tensorflow:examples/sec: 5.92862\n",
      "INFO:tensorflow:global_step/sec: 0.745003\n",
      "INFO:tensorflow:examples/sec: 5.96002\n",
      "INFO:tensorflow:global_step/sec: 0.740733\n",
      "INFO:tensorflow:examples/sec: 5.92586\n",
      "INFO:tensorflow:global_step/sec: 0.740625\n",
      "INFO:tensorflow:examples/sec: 5.925\n",
      "INFO:tensorflow:global_step/sec: 0.742251\n",
      "INFO:tensorflow:examples/sec: 5.93801\n",
      "INFO:tensorflow:global_step/sec: 0.744783\n",
      "INFO:tensorflow:examples/sec: 5.95827\n",
      "INFO:tensorflow:global_step/sec: 0.748265\n",
      "INFO:tensorflow:examples/sec: 5.98612\n",
      "INFO:tensorflow:global_step/sec: 0.750772\n",
      "INFO:tensorflow:examples/sec: 6.00617\n",
      "INFO:tensorflow:global_step/sec: 0.749954\n",
      "INFO:tensorflow:examples/sec: 5.99963\n",
      "INFO:tensorflow:global_step/sec: 0.747445\n",
      "INFO:tensorflow:examples/sec: 5.97956\n",
      "INFO:tensorflow:global_step/sec: 0.746279\n",
      "INFO:tensorflow:examples/sec: 5.97023\n",
      "INFO:tensorflow:global_step/sec: 0.747828\n",
      "INFO:tensorflow:examples/sec: 5.98262\n",
      "INFO:tensorflow:global_step/sec: 0.743618\n",
      "INFO:tensorflow:examples/sec: 5.94895\n",
      "INFO:tensorflow:global_step/sec: 0.746452\n",
      "INFO:tensorflow:examples/sec: 5.97161\n",
      "INFO:tensorflow:global_step/sec: 0.748037\n",
      "INFO:tensorflow:examples/sec: 5.9843\n",
      "INFO:tensorflow:global_step/sec: 0.741039\n",
      "INFO:tensorflow:examples/sec: 5.92831\n",
      "INFO:tensorflow:global_step/sec: 0.745112\n",
      "INFO:tensorflow:examples/sec: 5.9609\n",
      "INFO:tensorflow:global_step/sec: 0.744133\n",
      "INFO:tensorflow:examples/sec: 5.95307\n",
      "INFO:tensorflow:global_step/sec: 0.749022\n",
      "INFO:tensorflow:examples/sec: 5.99218\n",
      "INFO:tensorflow:global_step/sec: 0.740997\n",
      "INFO:tensorflow:examples/sec: 5.92797\n",
      "INFO:tensorflow:global_step/sec: 0.747211\n",
      "INFO:tensorflow:examples/sec: 5.97769\n",
      "INFO:tensorflow:global_step/sec: 0.746992\n",
      "INFO:tensorflow:examples/sec: 5.97594\n",
      "INFO:tensorflow:global_step/sec: 0.741966\n",
      "INFO:tensorflow:examples/sec: 5.93573\n",
      "INFO:tensorflow:global_step/sec: 0.740306\n",
      "INFO:tensorflow:examples/sec: 5.92245\n",
      "INFO:tensorflow:global_step/sec: 0.742915\n",
      "INFO:tensorflow:examples/sec: 5.94332\n",
      "INFO:tensorflow:global_step/sec: 0.749406\n",
      "INFO:tensorflow:examples/sec: 5.99524\n",
      "INFO:tensorflow:global_step/sec: 0.743976\n",
      "INFO:tensorflow:examples/sec: 5.95181\n",
      "INFO:tensorflow:global_step/sec: 0.742918\n",
      "INFO:tensorflow:examples/sec: 5.94334\n",
      "INFO:tensorflow:global_step/sec: 0.742591\n",
      "INFO:tensorflow:examples/sec: 5.94073\n",
      "INFO:tensorflow:global_step/sec: 0.742455\n",
      "INFO:tensorflow:examples/sec: 5.93964\n",
      "INFO:tensorflow:global_step/sec: 0.743509\n",
      "INFO:tensorflow:examples/sec: 5.94807\n",
      "INFO:tensorflow:global_step/sec: 0.749132\n",
      "INFO:tensorflow:examples/sec: 5.99305\n",
      "INFO:tensorflow:global_step/sec: 0.744535\n",
      "INFO:tensorflow:examples/sec: 5.95628\n",
      "INFO:tensorflow:global_step/sec: 0.745201\n",
      "INFO:tensorflow:examples/sec: 5.96161\n",
      "INFO:tensorflow:global_step/sec: 0.748603\n",
      "INFO:tensorflow:examples/sec: 5.98883\n",
      "INFO:tensorflow:global_step/sec: 0.74535\n",
      "INFO:tensorflow:examples/sec: 5.9628\n",
      "INFO:tensorflow:global_step/sec: 0.747792\n",
      "INFO:tensorflow:examples/sec: 5.98233\n",
      "INFO:tensorflow:global_step/sec: 0.747148\n",
      "INFO:tensorflow:examples/sec: 5.97718\n",
      "INFO:tensorflow:global_step/sec: 0.746778\n",
      "INFO:tensorflow:examples/sec: 5.97423\n",
      "INFO:tensorflow:global_step/sec: 0.742145\n",
      "INFO:tensorflow:examples/sec: 5.93716\n",
      "INFO:tensorflow:global_step/sec: 0.748451\n",
      "INFO:tensorflow:examples/sec: 5.98761\n",
      "INFO:tensorflow:global_step/sec: 0.74476\n",
      "INFO:tensorflow:examples/sec: 5.95808\n",
      "INFO:tensorflow:global_step/sec: 0.749824\n",
      "INFO:tensorflow:examples/sec: 5.99859\n",
      "INFO:tensorflow:global_step/sec: 0.745751\n",
      "INFO:tensorflow:examples/sec: 5.966\n",
      "INFO:tensorflow:global_step/sec: 0.745229\n",
      "INFO:tensorflow:examples/sec: 5.96183\n",
      "INFO:tensorflow:global_step/sec: 0.745797\n",
      "INFO:tensorflow:examples/sec: 5.96638\n",
      "INFO:tensorflow:global_step/sec: 0.743294\n",
      "INFO:tensorflow:examples/sec: 5.94636\n",
      "INFO:tensorflow:global_step/sec: 0.743519\n",
      "INFO:tensorflow:examples/sec: 5.94815\n",
      "INFO:tensorflow:global_step/sec: 0.74108\n",
      "INFO:tensorflow:examples/sec: 5.92864\n",
      "INFO:tensorflow:global_step/sec: 0.741887\n",
      "INFO:tensorflow:examples/sec: 5.9351\n",
      "INFO:tensorflow:global_step/sec: 0.747138\n",
      "INFO:tensorflow:examples/sec: 5.9771\n",
      "INFO:tensorflow:global_step/sec: 0.742121\n",
      "INFO:tensorflow:examples/sec: 5.93697\n",
      "INFO:tensorflow:global_step/sec: 0.740523\n",
      "INFO:tensorflow:examples/sec: 5.92419\n",
      "INFO:tensorflow:global_step/sec: 0.745659\n",
      "INFO:tensorflow:examples/sec: 5.96527\n",
      "INFO:tensorflow:global_step/sec: 0.742485\n",
      "INFO:tensorflow:examples/sec: 5.93988\n",
      "INFO:tensorflow:global_step/sec: 0.741385\n",
      "INFO:tensorflow:examples/sec: 5.93108\n",
      "INFO:tensorflow:global_step/sec: 0.74677\n",
      "INFO:tensorflow:examples/sec: 5.97416\n",
      "INFO:tensorflow:global_step/sec: 0.742881\n",
      "INFO:tensorflow:examples/sec: 5.94305\n",
      "INFO:tensorflow:global_step/sec: 0.748464\n",
      "INFO:tensorflow:examples/sec: 5.98771\n",
      "INFO:tensorflow:global_step/sec: 0.745905\n",
      "INFO:tensorflow:examples/sec: 5.96724\n",
      "INFO:tensorflow:global_step/sec: 0.750096\n",
      "INFO:tensorflow:examples/sec: 6.00077\n",
      "INFO:tensorflow:global_step/sec: 0.743304\n",
      "INFO:tensorflow:examples/sec: 5.94643\n",
      "INFO:tensorflow:global_step/sec: 0.746182\n",
      "INFO:tensorflow:examples/sec: 5.96946\n",
      "INFO:tensorflow:global_step/sec: 0.742419\n",
      "INFO:tensorflow:examples/sec: 5.93935\n",
      "INFO:tensorflow:global_step/sec: 0.745411\n",
      "INFO:tensorflow:examples/sec: 5.96329\n",
      "INFO:tensorflow:global_step/sec: 0.745017\n",
      "INFO:tensorflow:examples/sec: 5.96013\n",
      "INFO:tensorflow:global_step/sec: 0.741156\n",
      "INFO:tensorflow:examples/sec: 5.92925\n",
      "INFO:tensorflow:global_step/sec: 0.747016\n",
      "INFO:tensorflow:examples/sec: 5.97613\n",
      "INFO:tensorflow:global_step/sec: 0.744356\n",
      "INFO:tensorflow:examples/sec: 5.95485\n",
      "INFO:tensorflow:global_step/sec: 0.739016\n",
      "INFO:tensorflow:examples/sec: 5.91212\n",
      "INFO:tensorflow:global_step/sec: 0.747875\n",
      "INFO:tensorflow:examples/sec: 5.983\n",
      "INFO:tensorflow:global_step/sec: 0.745846\n",
      "INFO:tensorflow:examples/sec: 5.96677\n",
      "INFO:tensorflow:global_step/sec: 0.746467\n",
      "INFO:tensorflow:examples/sec: 5.97173\n",
      "INFO:tensorflow:global_step/sec: 0.74524\n",
      "INFO:tensorflow:examples/sec: 5.96192\n",
      "INFO:tensorflow:global_step/sec: 0.745498\n",
      "INFO:tensorflow:examples/sec: 5.96398\n",
      "INFO:tensorflow:global_step/sec: 0.745017\n",
      "INFO:tensorflow:examples/sec: 5.96013\n",
      "INFO:tensorflow:global_step/sec: 0.745851\n",
      "INFO:tensorflow:examples/sec: 5.96681\n",
      "INFO:tensorflow:global_step/sec: 0.752463\n",
      "INFO:tensorflow:examples/sec: 6.0197\n",
      "INFO:tensorflow:global_step/sec: 0.747618\n",
      "INFO:tensorflow:examples/sec: 5.98094\n",
      "INFO:tensorflow:global_step/sec: 0.747171\n",
      "INFO:tensorflow:examples/sec: 5.97737\n",
      "INFO:tensorflow:global_step/sec: 0.74145\n",
      "INFO:tensorflow:examples/sec: 5.9316\n",
      "INFO:tensorflow:global_step/sec: 0.749906\n",
      "INFO:tensorflow:examples/sec: 5.99925\n",
      "INFO:tensorflow:global_step/sec: 0.744582\n",
      "INFO:tensorflow:examples/sec: 5.95666\n",
      "INFO:tensorflow:global_step/sec: 0.744677\n",
      "INFO:tensorflow:examples/sec: 5.95742\n",
      "INFO:tensorflow:global_step/sec: 0.741888\n",
      "INFO:tensorflow:examples/sec: 5.9351\n",
      "INFO:tensorflow:global_step/sec: 0.742056\n",
      "INFO:tensorflow:examples/sec: 5.93645\n",
      "INFO:tensorflow:global_step/sec: 0.748198\n",
      "INFO:tensorflow:examples/sec: 5.98558\n",
      "INFO:tensorflow:global_step/sec: 0.745847\n",
      "INFO:tensorflow:examples/sec: 5.96678\n",
      "INFO:tensorflow:global_step/sec: 0.740597\n",
      "INFO:tensorflow:examples/sec: 5.92478\n",
      "INFO:tensorflow:global_step/sec: 0.743742\n",
      "INFO:tensorflow:examples/sec: 5.94993\n",
      "INFO:tensorflow:global_step/sec: 0.74389\n",
      "INFO:tensorflow:examples/sec: 5.95112\n",
      "INFO:tensorflow:global_step/sec: 0.747134\n",
      "INFO:tensorflow:examples/sec: 5.97707\n",
      "INFO:tensorflow:global_step/sec: 0.74716\n",
      "INFO:tensorflow:examples/sec: 5.97728\n",
      "INFO:tensorflow:global_step/sec: 0.744826\n",
      "INFO:tensorflow:examples/sec: 5.95861\n",
      "INFO:tensorflow:global_step/sec: 0.74022\n",
      "INFO:tensorflow:examples/sec: 5.92176\n",
      "INFO:tensorflow:global_step/sec: 0.750502\n",
      "INFO:tensorflow:examples/sec: 6.00401\n",
      "INFO:tensorflow:global_step/sec: 0.74829\n",
      "INFO:tensorflow:examples/sec: 5.98632\n",
      "INFO:tensorflow:global_step/sec: 0.748896\n",
      "INFO:tensorflow:examples/sec: 5.99117\n",
      "INFO:tensorflow:global_step/sec: 0.741744\n",
      "INFO:tensorflow:examples/sec: 5.93395\n",
      "INFO:tensorflow:global_step/sec: 0.745951\n",
      "INFO:tensorflow:examples/sec: 5.96761\n",
      "INFO:tensorflow:global_step/sec: 0.746679\n",
      "INFO:tensorflow:examples/sec: 5.97343\n",
      "INFO:tensorflow:global_step/sec: 0.738886\n",
      "INFO:tensorflow:examples/sec: 5.91109\n",
      "INFO:tensorflow:global_step/sec: 0.745983\n",
      "INFO:tensorflow:examples/sec: 5.96787\n",
      "INFO:tensorflow:global_step/sec: 0.753809\n",
      "INFO:tensorflow:examples/sec: 6.03047\n",
      "INFO:tensorflow:global_step/sec: 0.747698\n",
      "INFO:tensorflow:examples/sec: 5.98158\n",
      "INFO:tensorflow:global_step/sec: 0.747542\n",
      "INFO:tensorflow:examples/sec: 5.98033\n",
      "INFO:tensorflow:global_step/sec: 0.746781\n",
      "INFO:tensorflow:examples/sec: 5.97425\n",
      "INFO:tensorflow:global_step/sec: 0.747271\n",
      "INFO:tensorflow:examples/sec: 5.97817\n",
      "INFO:tensorflow:global_step/sec: 0.742877\n",
      "INFO:tensorflow:examples/sec: 5.94301\n",
      "INFO:tensorflow:global_step/sec: 0.744938\n",
      "INFO:tensorflow:examples/sec: 5.95951\n",
      "INFO:tensorflow:global_step/sec: 0.751108\n",
      "INFO:tensorflow:examples/sec: 6.00887\n",
      "INFO:tensorflow:global_step/sec: 0.744054\n",
      "INFO:tensorflow:examples/sec: 5.95243\n",
      "INFO:tensorflow:global_step/sec: 0.744743\n",
      "INFO:tensorflow:examples/sec: 5.95795\n",
      "INFO:tensorflow:global_step/sec: 0.743227\n",
      "INFO:tensorflow:examples/sec: 5.94582\n",
      "INFO:tensorflow:global_step/sec: 0.741418\n",
      "INFO:tensorflow:examples/sec: 5.93134\n",
      "INFO:tensorflow:global_step/sec: 0.747751\n",
      "INFO:tensorflow:examples/sec: 5.982\n",
      "INFO:tensorflow:global_step/sec: 0.752063\n",
      "INFO:tensorflow:examples/sec: 6.01651\n",
      "INFO:tensorflow:global_step/sec: 0.744385\n",
      "INFO:tensorflow:examples/sec: 5.95508\n",
      "INFO:tensorflow:global_step/sec: 0.751938\n",
      "INFO:tensorflow:examples/sec: 6.0155\n",
      "INFO:tensorflow:global_step/sec: 0.738526\n",
      "INFO:tensorflow:examples/sec: 5.9082\n",
      "INFO:tensorflow:global_step/sec: 0.741208\n",
      "INFO:tensorflow:examples/sec: 5.92966\n",
      "INFO:tensorflow:global_step/sec: 0.743982\n",
      "INFO:tensorflow:examples/sec: 5.95186\n",
      "INFO:tensorflow:global_step/sec: 0.743788\n",
      "INFO:tensorflow:examples/sec: 5.9503\n",
      "INFO:tensorflow:global_step/sec: 0.741727\n",
      "INFO:tensorflow:examples/sec: 5.93382\n",
      "INFO:tensorflow:global_step/sec: 0.745244\n",
      "INFO:tensorflow:examples/sec: 5.96195\n",
      "INFO:tensorflow:global_step/sec: 0.744735\n",
      "INFO:tensorflow:examples/sec: 5.95788\n",
      "INFO:tensorflow:global_step/sec: 0.743371\n",
      "INFO:tensorflow:examples/sec: 5.94697\n",
      "INFO:tensorflow:global_step/sec: 0.738918\n",
      "INFO:tensorflow:examples/sec: 5.91135\n",
      "INFO:tensorflow:global_step/sec: 0.740759\n",
      "INFO:tensorflow:examples/sec: 5.92607\n",
      "INFO:tensorflow:global_step/sec: 0.740605\n",
      "INFO:tensorflow:examples/sec: 5.92484\n",
      "INFO:tensorflow:global_step/sec: 0.748406\n",
      "INFO:tensorflow:examples/sec: 5.98724\n",
      "INFO:tensorflow:global_step/sec: 0.749348\n",
      "INFO:tensorflow:examples/sec: 5.99479\n",
      "INFO:tensorflow:global_step/sec: 0.744275\n",
      "INFO:tensorflow:examples/sec: 5.9542\n",
      "INFO:tensorflow:global_step/sec: 0.745365\n",
      "INFO:tensorflow:examples/sec: 5.96292\n",
      "INFO:tensorflow:global_step/sec: 0.747276\n",
      "INFO:tensorflow:examples/sec: 5.97821\n",
      "INFO:tensorflow:global_step/sec: 0.750368\n",
      "INFO:tensorflow:examples/sec: 6.00295\n",
      "INFO:tensorflow:global_step/sec: 0.740025\n",
      "INFO:tensorflow:examples/sec: 5.9202\n",
      "INFO:tensorflow:global_step/sec: 0.745113\n",
      "INFO:tensorflow:examples/sec: 5.96091\n",
      "INFO:tensorflow:global_step/sec: 0.744264\n",
      "INFO:tensorflow:examples/sec: 5.95411\n",
      "INFO:tensorflow:global_step/sec: 0.747716\n",
      "INFO:tensorflow:examples/sec: 5.98173\n",
      "INFO:tensorflow:global_step/sec: 0.743531\n",
      "INFO:tensorflow:examples/sec: 5.94825\n",
      "INFO:tensorflow:global_step/sec: 0.743958\n",
      "INFO:tensorflow:examples/sec: 5.95166\n",
      "INFO:tensorflow:global_step/sec: 0.748293\n",
      "INFO:tensorflow:examples/sec: 5.98634\n",
      "INFO:tensorflow:global_step/sec: 0.743894\n",
      "INFO:tensorflow:examples/sec: 5.95115\n",
      "INFO:tensorflow:global_step/sec: 0.745261\n",
      "INFO:tensorflow:examples/sec: 5.96209\n",
      "INFO:tensorflow:global_step/sec: 0.750062\n",
      "INFO:tensorflow:examples/sec: 6.0005\n",
      "INFO:tensorflow:global_step/sec: 0.742289\n",
      "INFO:tensorflow:examples/sec: 5.93831\n",
      "INFO:tensorflow:global_step/sec: 0.748705\n",
      "INFO:tensorflow:examples/sec: 5.98964\n",
      "INFO:tensorflow:global_step/sec: 0.746884\n",
      "INFO:tensorflow:examples/sec: 5.97507\n",
      "INFO:tensorflow:global_step/sec: 0.744737\n",
      "INFO:tensorflow:examples/sec: 5.9579\n",
      "INFO:tensorflow:global_step/sec: 0.749124\n",
      "INFO:tensorflow:examples/sec: 5.99299\n",
      "INFO:tensorflow:global_step/sec: 0.751889\n",
      "INFO:tensorflow:examples/sec: 6.01511\n",
      "INFO:tensorflow:global_step/sec: 0.741715\n",
      "INFO:tensorflow:examples/sec: 5.93372\n",
      "INFO:tensorflow:global_step/sec: 0.747283\n",
      "INFO:tensorflow:examples/sec: 5.97826\n",
      "INFO:tensorflow:global_step/sec: 0.742056\n",
      "INFO:tensorflow:examples/sec: 5.93645\n",
      "INFO:tensorflow:global_step/sec: 0.752022\n",
      "INFO:tensorflow:examples/sec: 6.01617\n",
      "INFO:tensorflow:global_step/sec: 0.741823\n",
      "INFO:tensorflow:examples/sec: 5.93458\n",
      "INFO:tensorflow:global_step/sec: 0.743315\n",
      "INFO:tensorflow:examples/sec: 5.94652\n",
      "INFO:tensorflow:global_step/sec: 0.750154\n",
      "INFO:tensorflow:examples/sec: 6.00123\n",
      "INFO:tensorflow:global_step/sec: 0.743511\n",
      "INFO:tensorflow:examples/sec: 5.94809\n",
      "INFO:tensorflow:global_step/sec: 0.746316\n",
      "INFO:tensorflow:examples/sec: 5.97053\n",
      "INFO:tensorflow:global_step/sec: 0.747151\n",
      "INFO:tensorflow:examples/sec: 5.97721\n",
      "INFO:tensorflow:global_step/sec: 0.744851\n",
      "INFO:tensorflow:examples/sec: 5.95881\n",
      "INFO:tensorflow:global_step/sec: 0.74488\n",
      "INFO:tensorflow:examples/sec: 5.95904\n",
      "INFO:tensorflow:global_step/sec: 0.7433\n",
      "INFO:tensorflow:examples/sec: 5.9464\n",
      "INFO:tensorflow:global_step/sec: 0.745604\n",
      "INFO:tensorflow:examples/sec: 5.96483\n",
      "INFO:tensorflow:global_step/sec: 0.745191\n",
      "INFO:tensorflow:examples/sec: 5.96153\n",
      "INFO:tensorflow:global_step/sec: 0.745052\n",
      "INFO:tensorflow:examples/sec: 5.96042\n",
      "INFO:tensorflow:global_step/sec: 0.747481\n",
      "INFO:tensorflow:examples/sec: 5.97985\n",
      "INFO:tensorflow:global_step/sec: 0.748001\n",
      "INFO:tensorflow:examples/sec: 5.98401\n",
      "INFO:tensorflow:global_step/sec: 0.748848\n",
      "INFO:tensorflow:examples/sec: 5.99079\n",
      "INFO:tensorflow:global_step/sec: 0.752106\n",
      "INFO:tensorflow:examples/sec: 6.01685\n",
      "INFO:tensorflow:global_step/sec: 0.743808\n",
      "INFO:tensorflow:examples/sec: 5.95046\n",
      "INFO:tensorflow:global_step/sec: 0.750306\n",
      "INFO:tensorflow:examples/sec: 6.00245\n",
      "INFO:tensorflow:global_step/sec: 0.744378\n",
      "INFO:tensorflow:examples/sec: 5.95502\n",
      "INFO:tensorflow:global_step/sec: 0.742044\n",
      "INFO:tensorflow:examples/sec: 5.93635\n",
      "INFO:tensorflow:global_step/sec: 0.750314\n",
      "INFO:tensorflow:examples/sec: 6.00251\n",
      "INFO:tensorflow:global_step/sec: 0.745313\n",
      "INFO:tensorflow:examples/sec: 5.96251\n",
      "INFO:tensorflow:global_step/sec: 0.743317\n",
      "INFO:tensorflow:examples/sec: 5.94653\n",
      "INFO:tensorflow:global_step/sec: 0.748746\n",
      "INFO:tensorflow:examples/sec: 5.98997\n",
      "INFO:tensorflow:global_step/sec: 0.749262\n",
      "INFO:tensorflow:examples/sec: 5.9941\n",
      "INFO:tensorflow:global_step/sec: 0.749495\n",
      "INFO:tensorflow:examples/sec: 5.99596\n",
      "INFO:tensorflow:global_step/sec: 0.748371\n",
      "INFO:tensorflow:examples/sec: 5.98697\n",
      "INFO:tensorflow:global_step/sec: 0.747642\n",
      "INFO:tensorflow:examples/sec: 5.98113\n",
      "INFO:tensorflow:global_step/sec: 0.743559\n",
      "INFO:tensorflow:examples/sec: 5.94847\n",
      "INFO:tensorflow:global_step/sec: 0.737422\n",
      "INFO:tensorflow:examples/sec: 5.89938\n",
      "INFO:tensorflow:global_step/sec: 0.746766\n",
      "INFO:tensorflow:examples/sec: 5.97413\n",
      "INFO:tensorflow:global_step/sec: 0.73669\n",
      "INFO:tensorflow:examples/sec: 5.89352\n",
      "INFO:tensorflow:global_step/sec: 0.751383\n",
      "INFO:tensorflow:examples/sec: 6.01106\n",
      "INFO:tensorflow:global_step/sec: 0.744829\n",
      "INFO:tensorflow:examples/sec: 5.95863\n",
      "INFO:tensorflow:global_step/sec: 0.744999\n",
      "INFO:tensorflow:examples/sec: 5.95999\n",
      "INFO:tensorflow:global_step/sec: 0.743693\n",
      "INFO:tensorflow:examples/sec: 5.94954\n",
      "INFO:tensorflow:global_step/sec: 0.7445\n",
      "INFO:tensorflow:examples/sec: 5.956\n",
      "INFO:tensorflow:global_step/sec: 0.74759\n",
      "INFO:tensorflow:examples/sec: 5.98072\n",
      "INFO:tensorflow:global_step/sec: 0.742994\n",
      "INFO:tensorflow:examples/sec: 5.94395\n",
      "INFO:tensorflow:global_step/sec: 0.74872\n",
      "INFO:tensorflow:examples/sec: 5.98976\n",
      "INFO:tensorflow:global_step/sec: 0.747086\n",
      "INFO:tensorflow:examples/sec: 5.97669\n",
      "INFO:tensorflow:global_step/sec: 0.742698\n",
      "INFO:tensorflow:examples/sec: 5.94158\n",
      "INFO:tensorflow:global_step/sec: 0.743507\n",
      "INFO:tensorflow:examples/sec: 5.94806\n",
      "INFO:tensorflow:global_step/sec: 0.747347\n",
      "INFO:tensorflow:examples/sec: 5.97877\n",
      "INFO:tensorflow:global_step/sec: 0.742794\n",
      "INFO:tensorflow:examples/sec: 5.94236\n",
      "INFO:tensorflow:global_step/sec: 0.751323\n",
      "INFO:tensorflow:examples/sec: 6.01058\n",
      "INFO:tensorflow:global_step/sec: 0.741588\n",
      "INFO:tensorflow:examples/sec: 5.93271\n",
      "INFO:tensorflow:global_step/sec: 0.741856\n",
      "INFO:tensorflow:examples/sec: 5.93485\n",
      "INFO:tensorflow:global_step/sec: 0.748076\n",
      "INFO:tensorflow:examples/sec: 5.98461\n",
      "INFO:tensorflow:global_step/sec: 0.743711\n",
      "INFO:tensorflow:examples/sec: 5.94969\n",
      "INFO:tensorflow:global_step/sec: 0.748177\n",
      "INFO:tensorflow:examples/sec: 5.98541\n",
      "INFO:tensorflow:global_step/sec: 0.742038\n",
      "INFO:tensorflow:examples/sec: 5.9363\n",
      "INFO:tensorflow:global_step/sec: 0.742772\n",
      "INFO:tensorflow:examples/sec: 5.94218\n",
      "INFO:tensorflow:global_step/sec: 0.744436\n",
      "INFO:tensorflow:examples/sec: 5.95549\n",
      "INFO:tensorflow:global_step/sec: 0.742009\n",
      "INFO:tensorflow:examples/sec: 5.93607\n",
      "INFO:tensorflow:global_step/sec: 0.745957\n",
      "INFO:tensorflow:examples/sec: 5.96766\n",
      "INFO:tensorflow:global_step/sec: 0.744614\n",
      "INFO:tensorflow:examples/sec: 5.95691\n",
      "INFO:tensorflow:global_step/sec: 0.744666\n",
      "INFO:tensorflow:examples/sec: 5.95733\n",
      "INFO:tensorflow:global_step/sec: 0.74435\n",
      "INFO:tensorflow:examples/sec: 5.9548\n",
      "INFO:tensorflow:global_step/sec: 0.74749\n",
      "INFO:tensorflow:examples/sec: 5.97992\n",
      "INFO:tensorflow:global_step/sec: 0.744005\n",
      "INFO:tensorflow:examples/sec: 5.95204\n",
      "INFO:tensorflow:global_step/sec: 0.744467\n",
      "INFO:tensorflow:examples/sec: 5.95573\n",
      "INFO:tensorflow:global_step/sec: 0.74399\n",
      "INFO:tensorflow:examples/sec: 5.95192\n",
      "INFO:tensorflow:global_step/sec: 0.746873\n",
      "INFO:tensorflow:examples/sec: 5.97499\n",
      "INFO:tensorflow:global_step/sec: 0.749462\n",
      "INFO:tensorflow:examples/sec: 5.99569\n",
      "INFO:tensorflow:global_step/sec: 0.749324\n",
      "INFO:tensorflow:examples/sec: 5.99459\n",
      "INFO:tensorflow:global_step/sec: 0.746677\n",
      "INFO:tensorflow:examples/sec: 5.97342\n",
      "INFO:tensorflow:global_step/sec: 0.746842\n",
      "INFO:tensorflow:examples/sec: 5.97473\n",
      "INFO:tensorflow:global_step/sec: 0.752477\n",
      "INFO:tensorflow:examples/sec: 6.01982\n",
      "INFO:tensorflow:global_step/sec: 0.743518\n",
      "INFO:tensorflow:examples/sec: 5.94814\n",
      "INFO:tensorflow:global_step/sec: 0.75242\n",
      "INFO:tensorflow:examples/sec: 6.01936\n",
      "INFO:tensorflow:global_step/sec: 0.743146\n",
      "INFO:tensorflow:examples/sec: 5.94517\n",
      "INFO:tensorflow:global_step/sec: 0.750992\n",
      "INFO:tensorflow:examples/sec: 6.00794\n",
      "INFO:tensorflow:global_step/sec: 0.748042\n",
      "INFO:tensorflow:examples/sec: 5.98434\n",
      "INFO:tensorflow:global_step/sec: 0.742501\n",
      "INFO:tensorflow:examples/sec: 5.94001\n",
      "INFO:tensorflow:global_step/sec: 0.750636\n",
      "INFO:tensorflow:examples/sec: 6.00509\n",
      "INFO:tensorflow:global_step/sec: 0.741179\n",
      "INFO:tensorflow:examples/sec: 5.92944\n",
      "INFO:tensorflow:global_step/sec: 0.744751\n",
      "INFO:tensorflow:examples/sec: 5.95801\n",
      "INFO:tensorflow:global_step/sec: 0.74481\n",
      "INFO:tensorflow:examples/sec: 5.95848\n",
      "INFO:tensorflow:global_step/sec: 0.748996\n",
      "INFO:tensorflow:examples/sec: 5.99197\n",
      "INFO:tensorflow:global_step/sec: 0.746062\n",
      "INFO:tensorflow:examples/sec: 5.9685\n",
      "INFO:tensorflow:global_step/sec: 0.743413\n",
      "INFO:tensorflow:examples/sec: 5.94731\n",
      "INFO:tensorflow:global_step/sec: 0.747557\n",
      "INFO:tensorflow:examples/sec: 5.98045\n",
      "INFO:tensorflow:global_step/sec: 0.745064\n",
      "INFO:tensorflow:examples/sec: 5.96051\n",
      "INFO:tensorflow:global_step/sec: 0.746349\n",
      "INFO:tensorflow:examples/sec: 5.97079\n",
      "INFO:tensorflow:global_step/sec: 0.747911\n",
      "INFO:tensorflow:examples/sec: 5.98329\n",
      "INFO:tensorflow:global_step/sec: 0.74155\n",
      "INFO:tensorflow:examples/sec: 5.9324\n",
      "INFO:tensorflow:global_step/sec: 0.744753\n",
      "INFO:tensorflow:examples/sec: 5.95803\n",
      "INFO:tensorflow:global_step/sec: 0.74648\n",
      "INFO:tensorflow:examples/sec: 5.97184\n",
      "INFO:tensorflow:global_step/sec: 0.747355\n",
      "INFO:tensorflow:examples/sec: 5.97884\n",
      "INFO:tensorflow:global_step/sec: 0.745441\n",
      "INFO:tensorflow:examples/sec: 5.96353\n",
      "INFO:tensorflow:global_step/sec: 0.744003\n",
      "INFO:tensorflow:examples/sec: 5.95203\n",
      "INFO:tensorflow:global_step/sec: 0.747212\n",
      "INFO:tensorflow:examples/sec: 5.97769\n",
      "INFO:tensorflow:global_step/sec: 0.748215\n",
      "INFO:tensorflow:examples/sec: 5.98572\n",
      "INFO:tensorflow:global_step/sec: 0.745056\n",
      "INFO:tensorflow:examples/sec: 5.96045\n",
      "INFO:tensorflow:global_step/sec: 0.743544\n",
      "INFO:tensorflow:examples/sec: 5.94835\n",
      "INFO:tensorflow:global_step/sec: 0.745554\n",
      "INFO:tensorflow:examples/sec: 5.96443\n",
      "INFO:tensorflow:global_step/sec: 0.748712\n",
      "INFO:tensorflow:examples/sec: 5.9897\n",
      "INFO:tensorflow:global_step/sec: 0.746372\n",
      "INFO:tensorflow:examples/sec: 5.97098\n",
      "INFO:tensorflow:global_step/sec: 0.742001\n",
      "INFO:tensorflow:examples/sec: 5.93601\n",
      "INFO:tensorflow:global_step/sec: 0.743787\n",
      "INFO:tensorflow:examples/sec: 5.9503\n",
      "INFO:tensorflow:global_step/sec: 0.748758\n",
      "INFO:tensorflow:examples/sec: 5.99007\n",
      "INFO:tensorflow:global_step/sec: 0.744443\n",
      "INFO:tensorflow:examples/sec: 5.95554\n",
      "INFO:tensorflow:global_step/sec: 0.746158\n",
      "INFO:tensorflow:examples/sec: 5.96926\n",
      "INFO:tensorflow:global_step/sec: 0.7407\n",
      "INFO:tensorflow:examples/sec: 5.9256\n",
      "INFO:tensorflow:global_step/sec: 0.743463\n",
      "INFO:tensorflow:examples/sec: 5.94771\n",
      "INFO:tensorflow:global_step/sec: 0.742067\n",
      "INFO:tensorflow:examples/sec: 5.93654\n",
      "INFO:tensorflow:global_step/sec: 0.735548\n",
      "INFO:tensorflow:examples/sec: 5.88439\n",
      "INFO:tensorflow:global_step/sec: 0.746458\n",
      "INFO:tensorflow:examples/sec: 5.97166\n",
      "INFO:tensorflow:global_step/sec: 0.744782\n",
      "INFO:tensorflow:examples/sec: 5.95826\n",
      "INFO:tensorflow:global_step/sec: 0.743624\n",
      "INFO:tensorflow:examples/sec: 5.94899\n",
      "INFO:tensorflow:global_step/sec: 0.739462\n",
      "INFO:tensorflow:examples/sec: 5.9157\n",
      "INFO:tensorflow:global_step/sec: 0.742883\n",
      "INFO:tensorflow:examples/sec: 5.94306\n",
      "INFO:tensorflow:global_step/sec: 0.742789\n",
      "INFO:tensorflow:examples/sec: 5.94231\n",
      "INFO:tensorflow:global_step/sec: 0.743111\n",
      "INFO:tensorflow:examples/sec: 5.94488\n",
      "INFO:tensorflow:global_step/sec: 0.741776\n",
      "INFO:tensorflow:examples/sec: 5.93421\n",
      "INFO:tensorflow:global_step/sec: 0.740538\n",
      "INFO:tensorflow:examples/sec: 5.9243\n",
      "INFO:tensorflow:global_step/sec: 0.745319\n",
      "INFO:tensorflow:examples/sec: 5.96255\n",
      "INFO:tensorflow:global_step/sec: 0.746163\n",
      "INFO:tensorflow:examples/sec: 5.96931\n",
      "INFO:tensorflow:global_step/sec: 0.740599\n",
      "INFO:tensorflow:examples/sec: 5.92479\n",
      "INFO:tensorflow:global_step/sec: 0.748113\n",
      "INFO:tensorflow:examples/sec: 5.9849\n",
      "INFO:tensorflow:global_step/sec: 0.741419\n",
      "INFO:tensorflow:examples/sec: 5.93135\n",
      "INFO:tensorflow:global_step/sec: 0.746673\n",
      "INFO:tensorflow:examples/sec: 5.97338\n",
      "INFO:tensorflow:global_step/sec: 0.740422\n",
      "INFO:tensorflow:examples/sec: 5.92338\n",
      "INFO:tensorflow:global_step/sec: 0.749028\n",
      "INFO:tensorflow:examples/sec: 5.99223\n",
      "INFO:tensorflow:global_step/sec: 0.743979\n",
      "INFO:tensorflow:examples/sec: 5.95183\n",
      "INFO:tensorflow:global_step/sec: 0.74833\n",
      "INFO:tensorflow:examples/sec: 5.98664\n",
      "INFO:tensorflow:global_step/sec: 0.75082\n",
      "INFO:tensorflow:examples/sec: 6.00656\n",
      "INFO:tensorflow:global_step/sec: 0.744619\n",
      "INFO:tensorflow:examples/sec: 5.95695\n",
      "INFO:tensorflow:global_step/sec: 0.747265\n",
      "INFO:tensorflow:examples/sec: 5.97812\n",
      "INFO:tensorflow:global_step/sec: 0.73887\n",
      "INFO:tensorflow:examples/sec: 5.91096\n",
      "INFO:tensorflow:global_step/sec: 0.742243\n",
      "INFO:tensorflow:examples/sec: 5.93794\n",
      "INFO:tensorflow:global_step/sec: 0.745894\n",
      "INFO:tensorflow:examples/sec: 5.96716\n",
      "INFO:tensorflow:global_step/sec: 0.745412\n",
      "INFO:tensorflow:examples/sec: 5.9633\n",
      "INFO:tensorflow:global_step/sec: 0.745291\n",
      "INFO:tensorflow:examples/sec: 5.96233\n",
      "INFO:tensorflow:global_step/sec: 0.745029\n",
      "INFO:tensorflow:examples/sec: 5.96023\n",
      "INFO:tensorflow:global_step/sec: 0.742806\n",
      "INFO:tensorflow:examples/sec: 5.94245\n",
      "INFO:tensorflow:global_step/sec: 0.739827\n",
      "INFO:tensorflow:examples/sec: 5.91862\n",
      "INFO:tensorflow:global_step/sec: 0.740718\n",
      "INFO:tensorflow:examples/sec: 5.92574\n",
      "INFO:tensorflow:global_step/sec: 0.741908\n",
      "INFO:tensorflow:examples/sec: 5.93526\n",
      "INFO:tensorflow:global_step/sec: 0.751019\n",
      "INFO:tensorflow:examples/sec: 6.00815\n",
      "INFO:tensorflow:global_step/sec: 0.7414\n",
      "INFO:tensorflow:examples/sec: 5.9312\n",
      "INFO:tensorflow:global_step/sec: 0.7423\n",
      "INFO:tensorflow:examples/sec: 5.9384\n",
      "INFO:tensorflow:global_step/sec: 0.737888\n",
      "INFO:tensorflow:examples/sec: 5.90311\n",
      "INFO:tensorflow:global_step/sec: 0.742236\n",
      "INFO:tensorflow:examples/sec: 5.93789\n",
      "INFO:tensorflow:global_step/sec: 0.751287\n",
      "INFO:tensorflow:examples/sec: 6.0103\n",
      "INFO:tensorflow:global_step/sec: 0.746927\n",
      "INFO:tensorflow:examples/sec: 5.97541\n",
      "INFO:tensorflow:global_step/sec: 0.746049\n",
      "INFO:tensorflow:examples/sec: 5.96839\n",
      "INFO:tensorflow:global_step/sec: 0.743035\n",
      "INFO:tensorflow:examples/sec: 5.94428\n",
      "INFO:tensorflow:global_step/sec: 0.744511\n",
      "INFO:tensorflow:examples/sec: 5.95609\n",
      "INFO:tensorflow:global_step/sec: 0.746272\n",
      "INFO:tensorflow:examples/sec: 5.97017\n",
      "INFO:tensorflow:global_step/sec: 0.742539\n",
      "INFO:tensorflow:examples/sec: 5.94031\n",
      "INFO:tensorflow:global_step/sec: 0.745719\n",
      "INFO:tensorflow:examples/sec: 5.96575\n",
      "INFO:tensorflow:global_step/sec: 0.740087\n",
      "INFO:tensorflow:examples/sec: 5.9207\n",
      "INFO:tensorflow:global_step/sec: 0.741641\n",
      "INFO:tensorflow:examples/sec: 5.93313\n",
      "INFO:tensorflow:global_step/sec: 0.742061\n",
      "INFO:tensorflow:examples/sec: 5.93649\n",
      "INFO:tensorflow:global_step/sec: 0.744184\n",
      "INFO:tensorflow:examples/sec: 5.95348\n",
      "INFO:tensorflow:global_step/sec: 0.744137\n",
      "INFO:tensorflow:examples/sec: 5.95309\n",
      "INFO:tensorflow:global_step/sec: 0.7495\n",
      "INFO:tensorflow:examples/sec: 5.996\n",
      "INFO:tensorflow:global_step/sec: 0.739389\n",
      "INFO:tensorflow:examples/sec: 5.91512\n",
      "INFO:tensorflow:global_step/sec: 0.746069\n",
      "INFO:tensorflow:examples/sec: 5.96855\n",
      "INFO:tensorflow:global_step/sec: 0.744226\n",
      "INFO:tensorflow:examples/sec: 5.95381\n",
      "INFO:tensorflow:global_step/sec: 0.742827\n",
      "INFO:tensorflow:examples/sec: 5.94262\n",
      "INFO:tensorflow:global_step/sec: 0.739017\n",
      "INFO:tensorflow:examples/sec: 5.91214\n",
      "INFO:tensorflow:global_step/sec: 0.745621\n",
      "INFO:tensorflow:examples/sec: 5.96497\n",
      "INFO:tensorflow:global_step/sec: 0.749782\n",
      "INFO:tensorflow:examples/sec: 5.99826\n",
      "INFO:tensorflow:global_step/sec: 0.746506\n",
      "INFO:tensorflow:examples/sec: 5.97204\n",
      "INFO:tensorflow:global_step/sec: 0.753056\n",
      "INFO:tensorflow:examples/sec: 6.02444\n",
      "INFO:tensorflow:global_step/sec: 0.752294\n",
      "INFO:tensorflow:examples/sec: 6.01836\n",
      "INFO:tensorflow:global_step/sec: 0.746728\n",
      "INFO:tensorflow:examples/sec: 5.97382\n",
      "INFO:tensorflow:global_step/sec: 0.746179\n",
      "INFO:tensorflow:examples/sec: 5.96943\n",
      "INFO:tensorflow:global_step/sec: 0.748823\n",
      "INFO:tensorflow:examples/sec: 5.99059\n",
      "INFO:tensorflow:global_step/sec: 0.742568\n",
      "INFO:tensorflow:examples/sec: 5.94054\n",
      "INFO:tensorflow:global_step/sec: 0.74868\n",
      "INFO:tensorflow:examples/sec: 5.98944\n",
      "INFO:tensorflow:global_step/sec: 0.744561\n",
      "INFO:tensorflow:examples/sec: 5.95649\n",
      "INFO:tensorflow:global_step/sec: 0.742628\n",
      "INFO:tensorflow:examples/sec: 5.94103\n",
      "INFO:tensorflow:global_step/sec: 0.746976\n",
      "INFO:tensorflow:examples/sec: 5.97581\n",
      "INFO:tensorflow:global_step/sec: 0.747445\n",
      "INFO:tensorflow:examples/sec: 5.97956\n",
      "INFO:tensorflow:global_step/sec: 0.74376\n",
      "INFO:tensorflow:examples/sec: 5.95008\n",
      "INFO:tensorflow:global_step/sec: 0.745712\n",
      "INFO:tensorflow:examples/sec: 5.9657\n",
      "INFO:tensorflow:global_step/sec: 0.748097\n",
      "INFO:tensorflow:examples/sec: 5.98478\n",
      "INFO:tensorflow:global_step/sec: 0.745537\n",
      "INFO:tensorflow:examples/sec: 5.9643\n",
      "INFO:tensorflow:global_step/sec: 0.749116\n",
      "INFO:tensorflow:examples/sec: 5.99293\n",
      "INFO:tensorflow:global_step/sec: 0.741624\n",
      "INFO:tensorflow:examples/sec: 5.93299\n",
      "INFO:tensorflow:global_step/sec: 0.747209\n",
      "INFO:tensorflow:examples/sec: 5.97767\n",
      "INFO:tensorflow:global_step/sec: 0.751215\n",
      "INFO:tensorflow:examples/sec: 6.00972\n",
      "INFO:tensorflow:global_step/sec: 0.744327\n",
      "INFO:tensorflow:examples/sec: 5.95461\n",
      "INFO:tensorflow:global_step/sec: 0.743578\n",
      "INFO:tensorflow:examples/sec: 5.94862\n",
      "INFO:tensorflow:global_step/sec: 0.748243\n",
      "INFO:tensorflow:examples/sec: 5.98594\n",
      "INFO:tensorflow:global_step/sec: 0.74731\n",
      "INFO:tensorflow:examples/sec: 5.97848\n",
      "INFO:tensorflow:global_step/sec: 0.744107\n",
      "INFO:tensorflow:examples/sec: 5.95285\n",
      "INFO:tensorflow:global_step/sec: 0.744551\n",
      "INFO:tensorflow:examples/sec: 5.95641\n",
      "INFO:tensorflow:global_step/sec: 0.746156\n",
      "INFO:tensorflow:examples/sec: 5.96925\n",
      "INFO:tensorflow:global_step/sec: 0.753202\n",
      "INFO:tensorflow:examples/sec: 6.02562\n",
      "INFO:tensorflow:global_step/sec: 0.738927\n",
      "INFO:tensorflow:examples/sec: 5.91142\n",
      "INFO:tensorflow:global_step/sec: 0.741912\n",
      "INFO:tensorflow:examples/sec: 5.93529\n",
      "INFO:tensorflow:global_step/sec: 0.747657\n",
      "INFO:tensorflow:examples/sec: 5.98126\n",
      "INFO:tensorflow:global_step/sec: 0.741645\n",
      "INFO:tensorflow:examples/sec: 5.93316\n",
      "INFO:tensorflow:global_step/sec: 0.742302\n",
      "INFO:tensorflow:examples/sec: 5.93842\n",
      "INFO:tensorflow:global_step/sec: 0.743417\n",
      "INFO:tensorflow:examples/sec: 5.94734\n",
      "INFO:tensorflow:global_step/sec: 0.742888\n",
      "INFO:tensorflow:examples/sec: 5.9431\n",
      "INFO:tensorflow:global_step/sec: 0.744384\n",
      "INFO:tensorflow:examples/sec: 5.95507\n",
      "INFO:tensorflow:global_step/sec: 0.746335\n",
      "INFO:tensorflow:examples/sec: 5.97068\n",
      "INFO:tensorflow:global_step/sec: 0.748976\n",
      "INFO:tensorflow:examples/sec: 5.99181\n",
      "INFO:tensorflow:global_step/sec: 0.741773\n",
      "INFO:tensorflow:examples/sec: 5.93418\n",
      "INFO:tensorflow:global_step/sec: 0.742781\n",
      "INFO:tensorflow:examples/sec: 5.94225\n",
      "INFO:tensorflow:global_step/sec: 0.74418\n",
      "INFO:tensorflow:examples/sec: 5.95344\n",
      "INFO:tensorflow:global_step/sec: 0.743756\n",
      "INFO:tensorflow:examples/sec: 5.95005\n",
      "INFO:tensorflow:global_step/sec: 0.747737\n",
      "INFO:tensorflow:examples/sec: 5.9819\n",
      "INFO:tensorflow:global_step/sec: 0.741336\n",
      "INFO:tensorflow:examples/sec: 5.93069\n",
      "INFO:tensorflow:global_step/sec: 0.740529\n",
      "INFO:tensorflow:examples/sec: 5.92424\n",
      "INFO:tensorflow:global_step/sec: 0.744792\n",
      "INFO:tensorflow:examples/sec: 5.95833\n",
      "INFO:tensorflow:global_step/sec: 0.739228\n",
      "INFO:tensorflow:examples/sec: 5.91382\n",
      "INFO:tensorflow:global_step/sec: 0.741287\n",
      "INFO:tensorflow:examples/sec: 5.9303\n",
      "INFO:tensorflow:global_step/sec: 0.744202\n",
      "INFO:tensorflow:examples/sec: 5.95362\n",
      "INFO:tensorflow:global_step/sec: 0.741923\n",
      "INFO:tensorflow:examples/sec: 5.93539\n",
      "INFO:tensorflow:global_step/sec: 0.745967\n",
      "INFO:tensorflow:examples/sec: 5.96774\n",
      "INFO:tensorflow:global_step/sec: 0.742865\n",
      "INFO:tensorflow:examples/sec: 5.94292\n",
      "INFO:tensorflow:global_step/sec: 0.745175\n",
      "INFO:tensorflow:examples/sec: 5.9614\n",
      "INFO:tensorflow:global_step/sec: 0.748609\n",
      "INFO:tensorflow:examples/sec: 5.98888\n",
      "INFO:tensorflow:global_step/sec: 0.745884\n",
      "INFO:tensorflow:examples/sec: 5.96707\n",
      "INFO:tensorflow:global_step/sec: 0.743505\n",
      "INFO:tensorflow:examples/sec: 5.94804\n",
      "INFO:tensorflow:global_step/sec: 0.742365\n",
      "INFO:tensorflow:examples/sec: 5.93892\n",
      "INFO:tensorflow:global_step/sec: 0.74275\n",
      "INFO:tensorflow:examples/sec: 5.942\n",
      "INFO:tensorflow:global_step/sec: 0.746256\n",
      "INFO:tensorflow:examples/sec: 5.97004\n",
      "INFO:tensorflow:global_step/sec: 0.742274\n",
      "INFO:tensorflow:examples/sec: 5.93819\n",
      "INFO:tensorflow:global_step/sec: 0.744324\n",
      "INFO:tensorflow:examples/sec: 5.95459\n",
      "INFO:tensorflow:global_step/sec: 0.74211\n",
      "INFO:tensorflow:examples/sec: 5.93688\n",
      "INFO:tensorflow:global_step/sec: 0.74414\n",
      "INFO:tensorflow:examples/sec: 5.95312\n",
      "INFO:tensorflow:global_step/sec: 0.740785\n",
      "INFO:tensorflow:examples/sec: 5.92628\n",
      "INFO:tensorflow:global_step/sec: 0.748585\n",
      "INFO:tensorflow:examples/sec: 5.98868\n",
      "INFO:tensorflow:global_step/sec: 0.746626\n",
      "INFO:tensorflow:examples/sec: 5.97301\n",
      "INFO:tensorflow:global_step/sec: 0.742182\n",
      "INFO:tensorflow:examples/sec: 5.93746\n",
      "INFO:tensorflow:global_step/sec: 0.747805\n",
      "INFO:tensorflow:examples/sec: 5.98244\n",
      "INFO:tensorflow:global_step/sec: 0.749426\n",
      "INFO:tensorflow:examples/sec: 5.99541\n",
      "INFO:tensorflow:global_step/sec: 0.746358\n",
      "INFO:tensorflow:examples/sec: 5.97087\n",
      "INFO:tensorflow:global_step/sec: 0.744008\n",
      "INFO:tensorflow:examples/sec: 5.95206\n",
      "INFO:tensorflow:global_step/sec: 0.744765\n",
      "INFO:tensorflow:examples/sec: 5.95812\n",
      "INFO:tensorflow:global_step/sec: 0.743253\n",
      "INFO:tensorflow:examples/sec: 5.94603\n",
      "INFO:tensorflow:global_step/sec: 0.744196\n",
      "INFO:tensorflow:examples/sec: 5.95357\n",
      "INFO:tensorflow:global_step/sec: 0.74724\n",
      "INFO:tensorflow:examples/sec: 5.97792\n",
      "INFO:tensorflow:global_step/sec: 0.739047\n",
      "INFO:tensorflow:examples/sec: 5.91237\n",
      "INFO:tensorflow:global_step/sec: 0.738881\n",
      "INFO:tensorflow:examples/sec: 5.91105\n",
      "INFO:tensorflow:global_step/sec: 0.743488\n",
      "INFO:tensorflow:examples/sec: 5.9479\n",
      "INFO:tensorflow:global_step/sec: 0.743892\n",
      "INFO:tensorflow:examples/sec: 5.95113\n",
      "INFO:tensorflow:global_step/sec: 0.74354\n",
      "INFO:tensorflow:examples/sec: 5.94832\n",
      "INFO:tensorflow:global_step/sec: 0.741118\n",
      "INFO:tensorflow:examples/sec: 5.92894\n",
      "INFO:tensorflow:global_step/sec: 0.74475\n",
      "INFO:tensorflow:examples/sec: 5.958\n",
      "INFO:tensorflow:global_step/sec: 0.742086\n",
      "INFO:tensorflow:examples/sec: 5.93669\n",
      "INFO:tensorflow:global_step/sec: 0.751083\n",
      "INFO:tensorflow:examples/sec: 6.00866\n",
      "INFO:tensorflow:global_step/sec: 0.749436\n",
      "INFO:tensorflow:examples/sec: 5.99549\n",
      "INFO:tensorflow:global_step/sec: 0.743745\n",
      "INFO:tensorflow:examples/sec: 5.94996\n",
      "INFO:tensorflow:global_step/sec: 0.742894\n",
      "INFO:tensorflow:examples/sec: 5.94315\n",
      "INFO:tensorflow:global_step/sec: 0.749138\n",
      "INFO:tensorflow:examples/sec: 5.9931\n",
      "INFO:tensorflow:global_step/sec: 0.749042\n",
      "INFO:tensorflow:examples/sec: 5.99233\n",
      "INFO:tensorflow:global_step/sec: 0.74127\n",
      "INFO:tensorflow:examples/sec: 5.93016\n",
      "INFO:tensorflow:global_step/sec: 0.745266\n",
      "INFO:tensorflow:examples/sec: 5.96212\n",
      "INFO:tensorflow:global_step/sec: 0.743337\n",
      "INFO:tensorflow:examples/sec: 5.94669\n",
      "INFO:tensorflow:global_step/sec: 0.741973\n",
      "INFO:tensorflow:examples/sec: 5.93579\n",
      "INFO:tensorflow:global_step/sec: 0.742763\n",
      "INFO:tensorflow:examples/sec: 5.9421\n",
      "INFO:tensorflow:global_step/sec: 0.743942\n",
      "INFO:tensorflow:examples/sec: 5.95153\n",
      "INFO:tensorflow:global_step/sec: 0.752033\n",
      "INFO:tensorflow:examples/sec: 6.01626\n",
      "INFO:tensorflow:global_step/sec: 0.741462\n",
      "INFO:tensorflow:examples/sec: 5.9317\n",
      "INFO:tensorflow:global_step/sec: 0.748109\n",
      "INFO:tensorflow:examples/sec: 5.98487\n",
      "INFO:tensorflow:global_step/sec: 0.748009\n",
      "INFO:tensorflow:examples/sec: 5.98407\n",
      "INFO:tensorflow:global_step/sec: 0.74108\n",
      "INFO:tensorflow:examples/sec: 5.92864\n",
      "INFO:tensorflow:global_step/sec: 0.7442\n",
      "INFO:tensorflow:examples/sec: 5.9536\n",
      "INFO:tensorflow:global_step/sec: 0.75134\n",
      "INFO:tensorflow:examples/sec: 6.01072\n",
      "INFO:tensorflow:global_step/sec: 0.748608\n",
      "INFO:tensorflow:examples/sec: 5.98886\n",
      "INFO:tensorflow:global_step/sec: 0.743786\n",
      "INFO:tensorflow:examples/sec: 5.95028\n",
      "INFO:tensorflow:global_step/sec: 0.74405\n",
      "INFO:tensorflow:examples/sec: 5.9524\n",
      "INFO:tensorflow:global_step/sec: 0.74284\n",
      "INFO:tensorflow:examples/sec: 5.94272\n",
      "INFO:tensorflow:global_step/sec: 0.74653\n",
      "INFO:tensorflow:examples/sec: 5.97224\n",
      "INFO:tensorflow:global_step/sec: 0.746859\n",
      "INFO:tensorflow:examples/sec: 5.97487\n",
      "INFO:tensorflow:global_step/sec: 0.744961\n",
      "INFO:tensorflow:examples/sec: 5.95969\n",
      "INFO:tensorflow:global_step/sec: 0.747831\n",
      "INFO:tensorflow:examples/sec: 5.98265\n",
      "INFO:tensorflow:global_step/sec: 0.746505\n",
      "INFO:tensorflow:examples/sec: 5.97204\n",
      "INFO:tensorflow:global_step/sec: 0.742045\n",
      "INFO:tensorflow:examples/sec: 5.93636\n",
      "INFO:tensorflow:global_step/sec: 0.75004\n",
      "INFO:tensorflow:examples/sec: 6.00032\n",
      "INFO:tensorflow:global_step/sec: 0.740664\n",
      "INFO:tensorflow:examples/sec: 5.92531\n",
      "INFO:tensorflow:global_step/sec: 0.739796\n",
      "INFO:tensorflow:examples/sec: 5.91837\n",
      "INFO:tensorflow:global_step/sec: 0.74622\n",
      "INFO:tensorflow:examples/sec: 5.96976\n",
      "INFO:tensorflow:global_step/sec: 0.748002\n",
      "INFO:tensorflow:examples/sec: 5.98402\n",
      "INFO:tensorflow:global_step/sec: 0.741498\n",
      "INFO:tensorflow:examples/sec: 5.93199\n",
      "INFO:tensorflow:global_step/sec: 0.7458\n",
      "INFO:tensorflow:examples/sec: 5.9664\n",
      "INFO:tensorflow:global_step/sec: 0.740733\n",
      "INFO:tensorflow:examples/sec: 5.92586\n",
      "INFO:tensorflow:global_step/sec: 0.745986\n",
      "INFO:tensorflow:examples/sec: 5.96789\n",
      "INFO:tensorflow:global_step/sec: 0.73796\n",
      "INFO:tensorflow:examples/sec: 5.90368\n",
      "INFO:tensorflow:global_step/sec: 0.737484\n",
      "INFO:tensorflow:examples/sec: 5.89987\n",
      "INFO:tensorflow:global_step/sec: 0.747852\n",
      "INFO:tensorflow:examples/sec: 5.98282\n",
      "INFO:tensorflow:global_step/sec: 0.744953\n",
      "INFO:tensorflow:examples/sec: 5.95962\n",
      "INFO:tensorflow:global_step/sec: 0.749739\n",
      "INFO:tensorflow:examples/sec: 5.99791\n",
      "INFO:tensorflow:global_step/sec: 0.746797\n",
      "INFO:tensorflow:examples/sec: 5.97438\n",
      "INFO:tensorflow:global_step/sec: 0.738321\n",
      "INFO:tensorflow:examples/sec: 5.90657\n",
      "INFO:tensorflow:global_step/sec: 0.744062\n",
      "INFO:tensorflow:examples/sec: 5.9525\n",
      "INFO:tensorflow:global_step/sec: 0.741922\n",
      "INFO:tensorflow:examples/sec: 5.93538\n",
      "INFO:tensorflow:global_step/sec: 0.74572\n",
      "INFO:tensorflow:examples/sec: 5.96576\n",
      "INFO:tensorflow:global_step/sec: 0.746515\n",
      "INFO:tensorflow:examples/sec: 5.97212\n",
      "INFO:tensorflow:global_step/sec: 0.744725\n",
      "INFO:tensorflow:examples/sec: 5.9578\n",
      "INFO:tensorflow:global_step/sec: 0.742676\n",
      "INFO:tensorflow:examples/sec: 5.94141\n",
      "INFO:tensorflow:global_step/sec: 0.743775\n",
      "INFO:tensorflow:examples/sec: 5.9502\n",
      "INFO:tensorflow:global_step/sec: 0.741268\n",
      "INFO:tensorflow:examples/sec: 5.93015\n",
      "INFO:tensorflow:global_step/sec: 0.747723\n",
      "INFO:tensorflow:examples/sec: 5.98179\n",
      "INFO:tensorflow:global_step/sec: 0.752947\n",
      "INFO:tensorflow:examples/sec: 6.02358\n",
      "INFO:tensorflow:global_step/sec: 0.746551\n",
      "INFO:tensorflow:examples/sec: 5.97241\n",
      "INFO:tensorflow:global_step/sec: 0.746226\n",
      "INFO:tensorflow:examples/sec: 5.96981\n",
      "INFO:tensorflow:global_step/sec: 0.746734\n",
      "INFO:tensorflow:examples/sec: 5.97387\n",
      "INFO:tensorflow:global_step/sec: 0.744931\n",
      "INFO:tensorflow:examples/sec: 5.95945\n",
      "INFO:tensorflow:global_step/sec: 0.749923\n",
      "INFO:tensorflow:examples/sec: 5.99938\n",
      "INFO:tensorflow:global_step/sec: 0.74425\n",
      "INFO:tensorflow:examples/sec: 5.954\n",
      "INFO:tensorflow:global_step/sec: 0.743378\n",
      "INFO:tensorflow:examples/sec: 5.94703\n",
      "INFO:tensorflow:global_step/sec: 0.741781\n",
      "INFO:tensorflow:examples/sec: 5.93425\n",
      "INFO:tensorflow:global_step/sec: 0.750137\n",
      "INFO:tensorflow:examples/sec: 6.0011\n",
      "INFO:tensorflow:global_step/sec: 0.744498\n",
      "INFO:tensorflow:examples/sec: 5.95598\n",
      "INFO:tensorflow:global_step/sec: 0.737849\n",
      "INFO:tensorflow:examples/sec: 5.90279\n",
      "INFO:tensorflow:global_step/sec: 0.750453\n",
      "INFO:tensorflow:examples/sec: 6.00363\n",
      "INFO:tensorflow:global_step/sec: 0.741782\n",
      "INFO:tensorflow:examples/sec: 5.93426\n",
      "INFO:tensorflow:global_step/sec: 0.746063\n",
      "INFO:tensorflow:examples/sec: 5.9685\n",
      "INFO:tensorflow:global_step/sec: 0.739383\n",
      "INFO:tensorflow:examples/sec: 5.91506\n",
      "INFO:tensorflow:global_step/sec: 0.741841\n",
      "INFO:tensorflow:examples/sec: 5.93473\n",
      "INFO:tensorflow:global_step/sec: 0.741843\n",
      "INFO:tensorflow:examples/sec: 5.93474\n",
      "INFO:tensorflow:global_step/sec: 0.742998\n",
      "INFO:tensorflow:examples/sec: 5.94398\n",
      "INFO:tensorflow:global_step/sec: 0.742014\n",
      "INFO:tensorflow:examples/sec: 5.93611\n",
      "INFO:tensorflow:global_step/sec: 0.747712\n",
      "INFO:tensorflow:examples/sec: 5.9817\n",
      "INFO:tensorflow:global_step/sec: 0.740291\n",
      "INFO:tensorflow:examples/sec: 5.92233\n",
      "INFO:tensorflow:global_step/sec: 0.745029\n",
      "INFO:tensorflow:examples/sec: 5.96023\n",
      "INFO:tensorflow:global_step/sec: 0.745696\n",
      "INFO:tensorflow:examples/sec: 5.96556\n",
      "INFO:tensorflow:global_step/sec: 0.746832\n",
      "INFO:tensorflow:examples/sec: 5.97465\n",
      "INFO:tensorflow:global_step/sec: 0.742881\n",
      "INFO:tensorflow:examples/sec: 5.94305\n",
      "INFO:tensorflow:global_step/sec: 0.749174\n",
      "INFO:tensorflow:examples/sec: 5.99339\n",
      "INFO:tensorflow:global_step/sec: 0.748046\n",
      "INFO:tensorflow:examples/sec: 5.98437\n",
      "INFO:tensorflow:global_step/sec: 0.744473\n",
      "INFO:tensorflow:examples/sec: 5.95578\n",
      "INFO:tensorflow:global_step/sec: 0.745902\n",
      "INFO:tensorflow:examples/sec: 5.96721\n",
      "INFO:tensorflow:Saving checkpoints for 111200 into t5-base-summary/model.ckpt.\n",
      "INFO:tensorflow:Before Save.\n",
      "INFO:tensorflow:About to write a checkpoint\n",
      "INFO:tensorflow:Done writing checkpoint.\n",
      "INFO:tensorflow:Done with the session.\n",
      "INFO:tensorflow:Loss for final step: 0.0059747403.\n",
      "INFO:tensorflow:training_loop marked as finished\n"
     ]
    }
   ],
   "source": [
    "model.finetune(\n",
    "    mixture_or_task_name=\"summary_bahasa\",\n",
    "    pretrained_model_dir='t5-base-bahasa',\n",
    "    finetune_steps=1000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(CNN) - Republik Sudan Selatan yang baru merdeka telah mengumumkan kematian pemimpin pemberontak terkemuka. Lt. Gen. George Athor dan salah seorang tenteranya terbunuh dalam pertempuran dengan tentera negara itu, Naib Presiden Riek Machar kepada wartawan Selasa. Dia berkata mereka telah menyusup ke Sudan Selatan dalam perjalanan keluar dari Rwanda melalui kawasan sempadan timur Republik Demokratik Congo dan Uganda. Tuntutan ini belum disahkan oleh badan bebas. Seorang komander pembangkang Tentera Pembebasan Rakyat Sudan, atau Spla, Athor memberontak setelah kalah dalam pemilihan gubernur pada musim bunga tahun 2010. Dia menyeru banyak pertempuran yang telah dirancang setelah pengumuman kekalahannya. \"Di bawah spanduk Gerakan Demokratik Sudan Selatan (SSDM) dan sayap tenteranya, Tentera Sudan Selatan, Gen. Athor dianggap sebagai pemberontak pemilihan pasca pemilihan yang paling kuat\" kata penyelidikan bebas yang berasaskan Geneva dalam laporannya pada tahun 2011 tentang apa kemudian Sudan selatan. Survei Arms Kecil mengatakan bahawa walaupun kedua-dua kerajaan Sudan Selatan dan Spla mendakwa bahawa Athor mendapat sokongan dari pemerintah Sudan di Khartoum, tidak ada bukti bebas untuk menyokong tuntutan ini. Kematian Athor bukan sebahagian daripada rancangan kerajaan, kerana kerajaan telah mengisytiharkan pengampunan umum kepada semua kumpulan pemberontak di Sudan Selatan, kata wakil presiden kepada wartawan. Kerajaan menarik semua pengikut Athor untuk mengamati pengampunan, yang diumumkan pada 9 Julai ketika negara merayakan kemerdekaannya dari Sudan. John Prendergast, pengasas Projek Cukup, mengatakan kematian Athor yang dilaporkan \"menyoroti perlunya segera dengan kerajaan Sudan Selatan, dengan sokongan antarabangsa, mesti menangani perpecahan antara kaum di Selatan. Seorang lagi Athor akan muncul esok kecuali perkembangan sebenar dibuat dalam menyediakan peluang politik dan ekonomi yang merasa terpinggir dalam proses kemerdekaan. \" Projek Cukup berfungsi untuk mendedahkan genosida dan jenayah terhadap kemanusiaan.\\tNaib Presiden Selatan Sudan mengumumkan kematian Lt Gen Gen. George Athor. Tuntutan itu belum disahkan oleh badan bebas. Athor memberontak selepas kehilangan pemilihan gubernur pada musim bunga tahun 2010'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('cnn-summarization.tsv') as fopen:\n",
    "    data = fopen.read().split('\\n')\n",
    "    \n",
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.io.gfile.GFile('test.txt', \"w\") as f:\n",
    "    f.write(\"ringkasan: %s\\n\" % data[0].split('\\t')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 't5-base-summary', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "isolate_session_state: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': None, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7f3158124d68>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running infer on CPU\n",
      "INFO:tensorflow:feature inputs : Tensor(\"Reshape:0\", shape=(1, 8, 1024), dtype=int32)\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable decoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 589824       slice_size 589824       Shape[heads=768, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 589824       slice_size 589824       Shape[d_model=768, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 2359296      slice_size 2359296      Shape[d_model=768, d_ff=3072]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 2359296      slice_size 2359296      Shape[d_ff=3072, d_model=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/layer_norm/scale                 size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable encoder/final_layer_norm/scale                               size 768          slice_size 768          Shape[d_model=768]                                          \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 24674304     slice_size 24674304     Shape[vocab=32128, d_model=768]                             \n",
      "INFO:tensorflow:Trainable Variables            count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:All Variables                  count: 257     Total size: 222903552        Total slice_size: 222903552      \n",
      "INFO:tensorflow:Counters:\n",
      "allconcat: 1.64e+04\n",
      " allconcat/1: 1.64e+04\n",
      "  allconcat/1/reshape_op: 1.64e+04\n",
      "allreduce: 3.9e+08\n",
      " allreduce/[0]: 3.9e+08\n",
      "  allreduce/[0]/einsum_op: 3.9e+08\n",
      "  allreduce/[0]/reduce_op: 16\n",
      " allreduce/[1]: 1\n",
      "  allreduce/[1]/reduce_op: 1\n",
      "einsum: 2.71e+12\n",
      "einsum_unique: 2.71e+12\n",
      "output: 3.57e+10\n",
      " output/AddOperation: 1.27e+10\n",
      " output/BinaryOpWithBroadcasting: 1.51e+08\n",
      " output/Constant: 1.51e+08\n",
      " output/EinsumOperation: 7.32e+09\n",
      " output/ImportOperation: 8.28e+03\n",
      " output/MinMaxOperation: 3.78e+07\n",
      " output/OneHotOperation: 1.33e+09\n",
      " output/RangeOperation: 2.05e+03\n",
      " output/ReduceOperation: 7.59e+06\n",
      " output/ReshapeOperation: 1.06e+09\n",
      " output/ScalarAddOperation: 5.08e+07\n",
      " output/ScalarMultiplyOperation: 2.08e+08\n",
      " output/ShiftOperation: 8.19e+03\n",
      " output/SlicewiseOperation: 8.68e+09\n",
      " output/StopGradient: 3.62e+09\n",
      " output/Variable: 2.23e+08\n",
      " output/WhileLoopOperation: 1.51e+08\n",
      "output_unique: 3.57e+10\n",
      " output_unique/AddOperation: 1.27e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 1.51e+08\n",
      " output_unique/Constant: 1.51e+08\n",
      " output_unique/EinsumOperation: 7.32e+09\n",
      " output_unique/ImportOperation: 8.28e+03\n",
      " output_unique/MinMaxOperation: 3.78e+07\n",
      " output_unique/OneHotOperation: 1.33e+09\n",
      " output_unique/RangeOperation: 2.05e+03\n",
      " output_unique/ReduceOperation: 7.59e+06\n",
      " output_unique/ReshapeOperation: 1.06e+09\n",
      " output_unique/ScalarAddOperation: 5.08e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.08e+08\n",
      " output_unique/ShiftOperation: 8.19e+03\n",
      " output_unique/SlicewiseOperation: 8.68e+09\n",
      " output_unique/StopGradient: 3.62e+09\n",
      " output_unique/Variable: 2.23e+08\n",
      " output_unique/WhileLoopOperation: 1.51e+08\n",
      "variables: 2.23e+08\n",
      " variables/trainable: 2.23e+08\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from t5-base-summary/model.ckpt-111200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:decoded 0: b'ringkasan: (CNN) - Republik Sudan Selatan yang baru merdeka telah mengumumkan kematian pemimpin pemberontak terkemuka. Lt. Gen. George Athor dan salah seorang tenteranya terbunuh dalam pertempuran dengan tentera negara itu, Naib Presiden Riek Machar kepada wartawan Selasa. Dia berkata mereka telah menyusup ke Sudan Selatan dalam perjalanan keluar dari Rwanda melalui kawasan sempadan timur Republik Demokratik Congo dan Uganda. Tuntutan ini belum disahkan oleh badan bebas. Seorang komander pembangkang Tentera Pembebasan Rakyat Sudan, atau Spla, Athor memberontak setelah kalah dalam pemilihan gubernur pada musim bunga tahun 2010. Dia menyeru banyak pertempuran yang telah dirancang setelah pengumuman kekalahannya. \"Di bawah spanduk Gerakan Demokratik Sudan Selatan (SSDM) dan sayap tenteranya, Tentera Sudan Selatan, Gen. Athor dianggap sebagai pemberontak pemilihan pasca pemilihan yang paling kuat\" kata penyelidikan bebas yang berasaskan Geneva dalam laporannya pada tahun 2011 tentang apa kemudian Sudan selatan. Survei Arms Kecil mengatakan bahawa walaupun kedua-dua kerajaan Sudan Selatan dan Spla mendakwa bahawa Athor mendapat sokongan dari pemerintah Sudan di Khartoum, tidak ada bukti bebas untuk menyokong tuntutan ini. Kematian Athor bukan sebahagian daripada rancangan kerajaan, kerana kerajaan telah mengisytiharkan pengampunan umum kepada semua kumpulan pemberontak di Sudan Selatan, kata wakil presiden kepada wartawan. Kerajaan menarik semua pengikut Athor untuk mengamati pengampunan, yang diumumkan pada 9 Julai ketika negara merayakan kemerdekaannya dari Sudan. John Prendergast, pengasas Projek Cukup, mengatakan kematian Athor yang dilaporkan \"menyoroti perlunya segera dengan kerajaan Sudan Selatan, dengan sokongan antarabangsa, mesti menangani perpecahan antara kaum di Selatan. Seorang lagi Athor akan muncul esok kecuali perkembangan sebenar dibuat dalam menyediakan peluang politik dan ekonomi yang merasa terpinggir dalam proses kemerdekaan. \" Projek Cukup berfungsi untuk mendedahkan genosida dan jenayah terhadap kemanusiaan.'\n",
      "INFO:tensorflow:            -> b'Lt. Gen. George Athor dilantik sebagai salah seorang \"etnik terbaik dengan menewaskan Presiden\" kata Herbert Black. Ia juga merupakan pengusiran pemberontak tidak disahkan. Dia berkata pasukan tentera ini cuba mengelakkan satu pertempuran'\n",
      "INFO:tensorflow:decoded 1: b'ringkasan: (CNN) - Republik Sudan Selatan yang baru merdeka telah mengumumkan kematian pemimpin pemberontak terkemuka. Lt. Gen. George Athor dan salah seorang tenteranya terbunuh dalam pertempuran dengan tentera negara itu, Naib Presiden Riek Machar kepada wartawan Selasa. Dia berkata mereka telah menyusup ke Sudan Selatan dalam perjalanan keluar dari Rwanda melalui kawasan sempadan timur Republik Demokratik Congo dan Uganda. Tuntutan ini belum disahkan oleh badan bebas. Seorang komander pembangkang Tentera Pembebasan Rakyat Sudan, atau Spla, Athor memberontak setelah kalah dalam pemilihan gubernur pada musim bunga tahun 2010. Dia menyeru banyak pertempuran yang telah dirancang setelah pengumuman kekalahannya. \"Di bawah spanduk Gerakan Demokratik Sudan Selatan (SSDM) dan sayap tenteranya, Tentera Sudan Selatan, Gen. Athor dianggap sebagai pemberontak pemilihan pasca pemilihan yang paling kuat\" kata penyelidikan bebas yang berasaskan Geneva dalam laporannya pada tahun 2011 tentang apa kemudian Sudan selatan. Survei Arms Kecil mengatakan bahawa walaupun kedua-dua kerajaan Sudan Selatan dan Spla mendakwa bahawa Athor mendapat sokongan dari pemerintah Sudan di Khartoum, tidak ada bukti bebas untuk menyokong tuntutan ini. Kematian Athor bukan sebahagian daripada rancangan kerajaan, kerana kerajaan telah mengisytiharkan pengampunan umum kepada semua kumpulan pemberontak di Sudan Selatan, kata wakil presiden kepada wartawan. Kerajaan menarik semua pengikut Athor untuk mengamati pengampunan, yang diumumkan pada 9 Julai ketika negara merayakan kemerdekaannya dari Sudan. John Prendergast, pengasas Projek Cukup, mengatakan kematian Athor yang dilaporkan \"menyoroti perlunya segera dengan kerajaan Sudan Selatan, dengan sokongan antarabangsa, mesti menangani perpecahan antara kaum di Selatan. Seorang lagi Athor akan muncul esok kecuali perkembangan sebenar dibuat dalam menyediakan peluang politik dan ekonomi yang merasa terpinggir dalam proses kemerdekaan. \" Projek Cukup berfungsi untuk mendedahkan genosida dan jenayah terhadap kemanusiaan.'\n",
      "INFO:tensorflow:            -> b'Presiden Sper telah mengisytiharkan pilihan gubernur tahun 2010. Jeneral Rep. George Athor dan pemimpin pembangkang DrSU terbunuh dalam pertempuran. Austriapung mengatakan ia sesuai dengan pentadbiran Sudan Selatan serta dua kerajaan Sudan Selatan, Spla. Tetapi pasukan itu akan dihapuskan selepas orang lain yang terluka Setiausaha Negara Sudan. Matlamat pembunuhan Athor lebih penting termasuk organisasi pemberontak tetapi juga membiayai politik'\n",
      "INFO:tensorflow:decoded 2: b'ringkasan: (CNN) - Republik Sudan Selatan yang baru merdeka telah mengumumkan kematian pemimpin pemberontak terkemuka. Lt. Gen. George Athor dan salah seorang tenteranya terbunuh dalam pertempuran dengan tentera negara itu, Naib Presiden Riek Machar kepada wartawan Selasa. Dia berkata mereka telah menyusup ke Sudan Selatan dalam perjalanan keluar dari Rwanda melalui kawasan sempadan timur Republik Demokratik Congo dan Uganda. Tuntutan ini belum disahkan oleh badan bebas. Seorang komander pembangkang Tentera Pembebasan Rakyat Sudan, atau Spla, Athor memberontak setelah kalah dalam pemilihan gubernur pada musim bunga tahun 2010. Dia menyeru banyak pertempuran yang telah dirancang setelah pengumuman kekalahannya. \"Di bawah spanduk Gerakan Demokratik Sudan Selatan (SSDM) dan sayap tenteranya, Tentera Sudan Selatan, Gen. Athor dianggap sebagai pemberontak pemilihan pasca pemilihan yang paling kuat\" kata penyelidikan bebas yang berasaskan Geneva dalam laporannya pada tahun 2011 tentang apa kemudian Sudan selatan. Survei Arms Kecil mengatakan bahawa walaupun kedua-dua kerajaan Sudan Selatan dan Spla mendakwa bahawa Athor mendapat sokongan dari pemerintah Sudan di Khartoum, tidak ada bukti bebas untuk menyokong tuntutan ini. Kematian Athor bukan sebahagian daripada rancangan kerajaan, kerana kerajaan telah mengisytiharkan pengampunan umum kepada semua kumpulan pemberontak di Sudan Selatan, kata wakil presiden kepada wartawan. Kerajaan menarik semua pengikut Athor untuk mengamati pengampunan, yang diumumkan pada 9 Julai ketika negara merayakan kemerdekaannya dari Sudan. John Prendergast, pengasas Projek Cukup, mengatakan kematian Athor yang dilaporkan \"menyoroti perlunya segera dengan kerajaan Sudan Selatan, dengan sokongan antarabangsa, mesti menangani perpecahan antara kaum di Selatan. Seorang lagi Athor akan muncul esok kecuali perkembangan sebenar dibuat dalam menyediakan peluang politik dan ekonomi yang merasa terpinggir dalam proses kemerdekaan. \" Projek Cukup berfungsi untuk mendedahkan genosida dan jenayah terhadap kemanusiaan.'\n",
      "INFO:tensorflow:            -> b'Lt. Gen. George Athor, salah seorang tenteranya terbunuh dalam pertempuran. Mereka telah menyusup ke utara Rwanda melalui sempadan Timur Tengah. Pegawai pertahanan As mendakwa ia akan menyerang gabenor untuk Kongres Demokratik Osas. Sitting mengancam kematian, kata Pbb'\n",
      "INFO:tensorflow:decoded 4: b'ringkasan: (CNN) - Republik Sudan Selatan yang baru merdeka telah mengumumkan kematian pemimpin pemberontak terkemuka. Lt. Gen. George Athor dan salah seorang tenteranya terbunuh dalam pertempuran dengan tentera negara itu, Naib Presiden Riek Machar kepada wartawan Selasa. Dia berkata mereka telah menyusup ke Sudan Selatan dalam perjalanan keluar dari Rwanda melalui kawasan sempadan timur Republik Demokratik Congo dan Uganda. Tuntutan ini belum disahkan oleh badan bebas. Seorang komander pembangkang Tentera Pembebasan Rakyat Sudan, atau Spla, Athor memberontak setelah kalah dalam pemilihan gubernur pada musim bunga tahun 2010. Dia menyeru banyak pertempuran yang telah dirancang setelah pengumuman kekalahannya. \"Di bawah spanduk Gerakan Demokratik Sudan Selatan (SSDM) dan sayap tenteranya, Tentera Sudan Selatan, Gen. Athor dianggap sebagai pemberontak pemilihan pasca pemilihan yang paling kuat\" kata penyelidikan bebas yang berasaskan Geneva dalam laporannya pada tahun 2011 tentang apa kemudian Sudan selatan. Survei Arms Kecil mengatakan bahawa walaupun kedua-dua kerajaan Sudan Selatan dan Spla mendakwa bahawa Athor mendapat sokongan dari pemerintah Sudan di Khartoum, tidak ada bukti bebas untuk menyokong tuntutan ini. Kematian Athor bukan sebahagian daripada rancangan kerajaan, kerana kerajaan telah mengisytiharkan pengampunan umum kepada semua kumpulan pemberontak di Sudan Selatan, kata wakil presiden kepada wartawan. Kerajaan menarik semua pengikut Athor untuk mengamati pengampunan, yang diumumkan pada 9 Julai ketika negara merayakan kemerdekaannya dari Sudan. John Prendergast, pengasas Projek Cukup, mengatakan kematian Athor yang dilaporkan \"menyoroti perlunya segera dengan kerajaan Sudan Selatan, dengan sokongan antarabangsa, mesti menangani perpecahan antara kaum di Selatan. Seorang lagi Athor akan muncul esok kecuali perkembangan sebenar dibuat dalam menyediakan peluang politik dan ekonomi yang merasa terpinggir dalam proses kemerdekaan. \" Projek Cukup berfungsi untuk mendedahkan genosida dan jenayah terhadap kemanusiaan.'\n",
      "INFO:tensorflow:            -> b'Lt. Gen. George Athor, Cnn, terbunuh dalam pertempuran dengan tentera Afrika Selatan dan selatan Iran pada tahun 2010. Pemberontakal Parti Demokratik Sudan melabelkan Obama sebagai pemberontak pemilihan terhadap tindakan upaya. Presiden Nordloorg: Pemberontakgnan zaman tinggi \"menolak terlalu perkauman bawah satu level\". I Beet Software: Pencegahan genosida, jenayah terhadap kemanusiaan'\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    }
   ],
   "source": [
    "model.predict(\n",
    "    input_file='test.txt',\n",
    "    output_file='out.txt',\n",
    "    temperature=1.0,\n",
    "    sentencepiece_model_path='sp10m.cased.t5.model'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
