{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "from t5.data import preprocessors as prep\n",
    "import functools\n",
    "import t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = 'gs://mesolitica-general/t5-vocab/sp10m.cased.t5.model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dumping_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-general/t5-data/dumping-iium.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-news.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-parliament.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-pdf.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-watpadd.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-wiki.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['title', 'text'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('dumping_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'dumping_dataset',\n",
    "    dataset_fn = dumping_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = functools.partial(\n",
    "        t5.data.preprocessors.rekey,\n",
    "        key_map = {'inputs': None, 'targets': 'text'},\n",
    "    ),\n",
    "    token_preprocessor = t5.data.preprocessors.unsupervised,\n",
    "    sentencepiece_model_path = vocab,\n",
    "    metric_fns = [],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:unsupervised preprocessor got preprocessors=None; no preprocessing will be applied.\n",
      "{'inputs_plaintext': b'', 'inputs': array([1]), 'targets_plaintext': b'Ya bangsa itu lagi.', 'targets': array([1958, 1057,   24,   95,    3,    1])}\n",
      "{'inputs_plaintext': b'', 'inputs': array([1]), 'targets_plaintext': b'Kalau aku balik kerja lambat dek kerana timbunan kerja, dia sanggup tunggu aku dan balik kerja sekali.', 'targets': array([ 2002,  3359,  1069,   290,  9313, 10523,    69, 26169,   290,\n",
      "          14,    18,    13,  3709,  6839,  3359,    16,  1069,   290,\n",
      "         383,     3,     1])}\n",
      "{'inputs_plaintext': b'', 'inputs': array([1]), 'targets_plaintext': b'Konflik bermula bila mak mula terasa bahang sebab aku da mula tak bantu dalam belanja rumahtangga.', 'targets': array([12510,   348,  1889,  8468,   407, 13543,   547,   562,   873,\n",
      "        3359,  3476,   407,   379,  4242,    19,  5950, 18575,     3,\n",
      "           1])}\n",
      "{'inputs_plaintext': b'', 'inputs': array([1]), 'targets_plaintext': b'Bila dia ada, ada sahaja yang tak kena sebab saya rasakan dia tak suka saya dan salah satu sebabnya, dia ada perasaan cemburu kalau suami luangkan masa dengan saya.', 'targets': array([ 5974,    18,    13,    64,    14,    64,   304,    17,   379,\n",
      "        1868,   873,   113,   957,    54,    18,    13,   379,  2301,\n",
      "         113,    16,   201,    55, 11602,    14,    18,    13,    64,\n",
      "        2254, 17748,  1285,  2280,     1])}\n",
      "{'inputs_plaintext': b'', 'inputs': array([1]), 'targets_plaintext': b'Aku keliru.', 'targets': array([ 4491, 10484,     3,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('dumping_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def question_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-general/t5-data/qa-train.tsv',\n",
    "            'gs://mesolitica-general/t5-data/qa-validation.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def question_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['soalan: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('question_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'question_dataset',\n",
    "    dataset_fn = question_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [question_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'soalan: siapa yang menerjemahkan Alkitab King James ke Bahasa Inggeris?', 'inputs': array([ 2105,    50,  2746,    17,  5220,  2406,  2675,    54, 19079,\n",
      "        3400,  3574,    30,   741,   719,   275,     1]), 'targets_plaintext': b'47 cendekiawan, yang semuanya adalah anggota Gereja England', 'targets': array([ 3658, 13521,    14,    17,  3434,    35,   381,  4874,  2292,\n",
      "           1])}\n",
      "{'inputs_plaintext': b'soalan: siapakah gabenor puerto rico sekarang?', 'inputs': array([ 2105,    50, 16118, 10511,  6697,   254,   584,  9667,  1618,\n",
      "         769,   275,     1]), 'targets_plaintext': b'Alejandro Javier Garc\\xc3\\xada Padilla', 'targets': array([  209,  7029,    13, 16069, 31915,  4565,   823,     2,    13,\n",
      "        9314,  6471,     1])}\n",
      "{'inputs_plaintext': b'soalan: di mana pertempuran pertama revolusi texas berlaku?', 'inputs': array([2105,   50,   18,  119, 2815,  131, 5932, 2478, 1049,   13,   25,\n",
      "        216,  275,    1]), 'targets_plaintext': b'Gonzales', 'targets': array([16446,  9131,   329,     1])}\n",
      "{'inputs_plaintext': b'soalan: siapa yang menyanyikan lagu terakhir yang pernah saya tulis untuk anda?', 'inputs': array([2105,   50, 2746,   17, 8690,  573,  981,   17,  283,  113, 6717,\n",
      "         21,   63,   13,  275,    1]), 'targets_plaintext': b'David Cook', 'targets': array([ 3449, 14387,     1])}\n",
      "{'inputs_plaintext': b'soalan: siapa pengarang tunas bukan kawan?', 'inputs': array([ 2105,    50,  2746,  6078, 20505,    25,   149,  4818,   275,\n",
      "           1]), 'targets_plaintext': b'Christopher Paul Curtis', 'targets': array([13970,  3465,  7419,  2598,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('question_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pair_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-general/t5-data/dumping-iium-pair.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-news-pair.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-parliament-pair.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-pdf-pair.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-watpadd-pair.tsv',\n",
    "            'gs://mesolitica-general/t5-data/dumping-wiki-pair.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['text'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('pair_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'pair_dataset',\n",
    "    dataset_fn = pair_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [prep.next_sentence_prediction],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function neighboring_pairs.<locals>.<lambda> at 0x7f8a9c724ae8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7f8a9c724ae8>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n",
      "WARNING: Entity <function neighboring_pairs.<locals>.<lambda> at 0x7f8a9c724ae8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: Failed to parse source code of <function neighboring_pairs.<locals>.<lambda> at 0x7f8a9c724ae8>, which Python reported as:\n",
      "      lambda x: x[text_key], num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
      "\n",
      "If this is a lambda function, the error may be avoided by creating the lambda in a standalone statement.\n",
      "{'inputs_plaintext': b'nsp: Sangat menyedihkan bila saya dah siap masak berjenis2 masakan, nasi pulak masak lebih banyak dari biasa.. Tapi progress sekarang saya still seorang student..', 'inputs': array([ 2532,  7175,    50, 17205, 12863,  1889,   113,  4362,  1437,\n",
      "        5008,   181,  8129,   143,  3909,    14,  3705,   218,   139,\n",
      "        5008,    51,   127,    37,   634,     3,     3,  2987, 17583,\n",
      "         769,   113,  5205,   105, 11559,     3,     3,     1]), 'targets_plaintext': b'not_next', 'targets': array([ 699, 3835,  144, 2990,   81,    1])}\n",
      "{'inputs_plaintext': b'nsp: Aku present pakai Power Point semua bukti-bukti tu.. Power tak?.', 'inputs': array([ 2532,  7175,    50,  4491,  8859,  2516,  8383, 10843,   125,\n",
      "        1686,     7, 12756,  2693,     3,     3,  8383,   379,   275,\n",
      "           3,     1]), 'targets_plaintext': b'next', 'targets': array([7426,    1])}\n",
      "{'inputs_plaintext': b'nsp: Mak rasa, kemalangan tu semuanya berpunca daripada masalah penglihatan dia\".. Aku tak dedahkan identiti aku..', 'inputs': array([ 2532,  7175,    50,  4343,   957,    14,  1808,  2693,  3434,\n",
      "        3848,    47,   248,  7702,    18,    13,     6,     3,     3,\n",
      "        4491,   379, 15678,    54,  3495,  3359,     3,     3,     1]), 'targets_plaintext': b'not_next', 'targets': array([ 699, 3835,  144, 2990,   81,    1])}\n",
      "{'inputs_plaintext': b'nsp: Lastly pasal yang komen, \"ala hal kecik dik nanti kahwin lagi banyak masalah nanti kerja lagi banyak masalah\" yer kakak2 dan abang2 sekalian.. Tunggu punya tunggu tak juga muncul..', 'inputs': array([ 2532,  7175,    50, 12104,   375,  8759,    17,  4508,    14,\n",
      "          15,     6,   286,    13,   539,    30,  6428,    18,   139,\n",
      "        1992, 12522,    95,   127,   248,  1992,   290,    95,   127,\n",
      "         248,     6,    15,  7602,  9446,   143,    16,  7664,   143,\n",
      "        7294,     3,     3, 25788,  3453,  6839,   379,    42,   954,\n",
      "           3,     3,     1]), 'targets_plaintext': b'not_next', 'targets': array([ 699, 3835,  144, 2990,   81,    1])}\n",
      "{'inputs_plaintext': b'nsp: Kami akan cuba elak makan satu meja dengan ayah.. Memang tak boleh hadap muka ayah lama-lama..', 'inputs': array([ 2532,  7175,    50,  1791,    38,  1006, 12024,  1364,    55,\n",
      "        6213,    22,  6255,     3,     3,  6977,   379,    66,    15,\n",
      "       11684,  2055,  6255,   459,     7,  1603,    13,     3,     3,\n",
      "           1]), 'targets_plaintext': b'next', 'targets': array([7426,    1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('pair_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def news_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-general/t5-data/news-title.tsv',\n",
    "            'gs://mesolitica-general/t5-data/news-title2.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def news_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['tajuk: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('news_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'news_dataset',\n",
    "    dataset_fn = news_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [news_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'tajuk: BATU PAHAT: Dewan Rakyat sehingga kini belum menerima notis atau surat daripada Ahli Parlimen yang mengumumkan untuk menjadi bebas atau lompat parti. Yang Dipertuanya, Datuk Mohamad Ariff Md Yusof, berkata beliau hanya tahu ada beberapa Ahli Parlimen yang mengisytiharkan sebagai Ahli Parlimen Bebas dan keluar parti tertentu hanya menerusi media. \\xe2\\x80\\x9cSetakat ini, saya tiada senarai Ahli Parlimen yang isytihar bebas mahu pun melompat ke parti lain kerana belum menerima surat, notis atau dokumen bertulis daripada mereka seperti dipaparkan media. \\xe2\\x80\\x9cParlimen atau saya hanya dapat mengesahkan perkara ini apabila kami mendapat dokumen bertulis daripada mereka (Ahli Parlimen keluar parti). \\xe2\\x80\\x9cMungkin masih awal kerana sesi Dewan Rakyat tahun depan pun hanya akan bermula pada Mac depan. Sekiranya benar, Setiausaha Parlimen akan menentukan semula tempat duduk mereka (Ahli Parlimen keluar parti) di dalam dewan \\xe2\\x80\\x9d katanya. Beliau berkata demikian ketika merasmikan Program Outreach Parlimen dan Pameran Demokrasi Berparlimen anjuran Parlimen Batu Pahat di Dewan Jubli Intan, di sini, hari ini. Yang turut hadir, Timbalan Yang Dipertua Dewan Rakyat, Datuk Mohd Rashid Hasnon yang juga Ahli Parlimen Batu Pahat, Speaker Dewan Undangan Negeri (DUN) Johor, Suhaizan Kaiat dan Timbalan Speaker DUN Johor, Gan Peck Cheng. Selain itu, Ahli Parlimen Parit Sulong, Datuk Dr Noraini Ahmad yang juga Ketua Wanita UMNO turut hadir. Pada 14 Disember lalu, enam Ahli Parlimen UMNO mengisytiharkan meninggalkan parti keramat Melayu itu berkuat kuasa pada hari berkenaan. Ia membabitkan Ahli Parlimen Larut, Datuk Seri Hamzah Zainudin; Ahli Parlimen Hulu Terengganu, Datuk Rosol Wahid; Ahli Parlimen Mersing, Datuk Abdul Latiff Ahmad; Ahli Parlimen Tasik Gelugor, Datuk Shabudin Yahaya; Ahli Parlimen Sabak Bernam, Datuk Mohamad Fasiah Mohd Fakeh dan Ahli Parlimen Tanah Merah, Datuk Seri Ikmal Hisham Abdul Aziz Terdahulu, Mohamad Ariff merasmikan program itu yang pertama kali pernah dianjurkan Parlimen bagi mendekati rakyat, sekali gus memberi pemahaman fungsi sebenar institusi berkenaan. Sementara itu, Mohamad Ariff berkata, pihaknya tidak akan memperkenalkan peraturan baharu bagi mengawal selia kelakuan Ahli Parlimen yang bersidang di Dewan Rakyat. \\xe2\\x80\\x9cPeraturan sedia ada mencukupi untuk mengawal selia dan mendisiplinkan Ahli Parlimen. \\xe2\\x80\\x9cCuma perlu dikenakan penalti lebih tegas untuk kawal Ahli Parlimen antaranya mengusir keluar jika mereka terus berdegil dan melakukan provokasi melampau,\\xe2\\x80\\x9d katanya.', 'inputs': array([ 4509,    50, 12711, 29446,    50,   373,   523,   135,   210,\n",
      "         538,   263,  5262,    46,  1148,    47,   367,   333,    17,\n",
      "        1152,    21,    56,   939,    46,  7058,   258,     3,   124,\n",
      "         782,    26,    14,   106,   477,  8093,  2677,  2281,    14,\n",
      "          60,    71,   121,   881,    64,   103,   367,   333,    17,\n",
      "        4007,    41,   367,   333,  3573,    16,   784,   258,   753,\n",
      "         121,   474,   257,     3,    15,     2, 11089,    20,    14,\n",
      "         113,   431,  1588,   367,   333,    17, 20246,   939,   391,\n",
      "         393, 13275,    30,   258,    74,    69,   538,   263,  1148,\n",
      "          14,  5262,    46,  1825,  5494,    47,    45,    59,  8630,\n",
      "         257,     3,    15,     2, 29947,    46,   113,   121,    91,\n",
      "        2024,   273,    20,   169,   328,   231,  1825,  5494,    47,\n",
      "          45,    15,     4, 10634,   333,   784,   258,     5,     3,\n",
      "          15,     2, 12732,   145,   292,    69,  1333,   373,   523,\n",
      "          31,   632,   393,   121,    38,   348,    23,   471,   632,\n",
      "           3,  3427,  1076,    14,  1181,   333,    38,  1624,   196,\n",
      "         208,  2654,    45,    15,     4, 10634,   333,   784,   258,\n",
      "           5,    18,    19,  2759,    15,     2,    70,     3,    99,\n",
      "          60,   606,   110,  2948,   601,  8517,   591,  6115,   333,\n",
      "          16,  6877, 12012,   534,  2453,   440,   991,  3767,   333,\n",
      "        1028,  7272,    18,   373, 19936,  5767,    14,    18,   192,\n",
      "          14,    78,    20,     3,   124,   134,   922,    14,   552,\n",
      "         124,   782,   373,   523,    14,   106,   259,  4359,  3412,\n",
      "        6107,    17,    42,   367,   333,  1028,  7272,    14,  6740,\n",
      "         373,  1706,   156,    15,     4,  3849,     5,   429,    14,\n",
      "        9212,    13, 13010,  6741,    13,    81,    16,   552,  6740,\n",
      "        1915,   429,    14,  5552,  1099,  1875,  6995,     3,   317,\n",
      "          24,    14,   367,   333,  4495, 14534,    14,   106,   122,\n",
      "       12731,   342,    17,    42,   202,  1173,   400,   134,   922,\n",
      "           3,    67,   592,   673,   148,    14,   810,   367,   333,\n",
      "         400,  4007,  1360,   258, 16554,   142,    24,  4741,   284,\n",
      "          23,    78,   159,     3,   132, 13771,   367,   333, 18938,\n",
      "          14,   106,   188,  3891, 22000,   111,   367,   333,  2584,\n",
      "         588,    14,   106,  3269,  1388,  6565,   111,   367,   333,\n",
      "       13878,    14,   106,   288, 13596,   342,   111,   367,   333,\n",
      "        5628, 11544,    14,   106, 25307, 11468,   111,   367,   333,\n",
      "       13365, 12248,    14,   106,   477, 23972,   360,   259, 30021,\n",
      "          16,   367,   333,   792,  2965,    14,   106,   188, 27149,\n",
      "        6042,   288,  1502,  2724,    14,   477,  8093,  2948,   200,\n",
      "          24,    17,   131,   252,   283,  3364,   333,    43,  6485,\n",
      "         129,    14,   383,  1474,   150,  5390,  2104,  1212,   854,\n",
      "         159,     3,   565,    24,    14,   477,  8093,    60,    14,\n",
      "         930,    29,    38,  1638,  1448,   355,    43,  1722,  4652,\n",
      "       10492,   367,   333,    17,  9725,    18,   373,   523,     3,\n",
      "          15,     2,  9376,   757,    64,  2413,    21,  1722,  4652,\n",
      "          16,   970, 23399,    54,   367,   333,     3,    15,     2,\n",
      "       24298,   101,  1372,  4621,    51,  2231,    21,  7322,   367,\n",
      "         333,  1290, 12885,   784,   268,    45,   224, 29205,    16,\n",
      "         341, 10643,  6805,    14,     2,    70,     3,     1]), 'targets_plaintext': b'Dewan Rakyat belum terima surat Ahli Parlimen lompat parti', 'targets': array([ 373,  523,  538, 1238, 1148,  367,  333, 7058,  258,    1])}\n",
      "{'inputs_plaintext': b'tajuk: Jakarta, CNN Indonesia -- Partai Bulan Bintang (PBB) dan Partai Keadilan dan Persatuan Indonesia (PKPI) akan menggugat Komisi Pemilihan Umum (KPU) ke Badan Pengawas Pemilihan Umum (Bawaslu) karena tak diloloskan dalam tahap verifikasi partai politik peserta Pemilu 2019.\"Segera kami akan lakukan (gugatan) dan mungkin hari ini akan segera kami ajukan, karena tahapan-tahapan di KPU ini sangat cepat,\" ujar Sekretaris Jenderal PBB Afriansyah Noor, usai mengikuti rapat pleno Rekapitulasi Nasional Hasil Verifikasi Partai Politik Calon Peserta Pemilu 2019, di Jakarta, Sabtu (17/2).Ia mengatakan inti gugatan yang akan diajukan oleh pihaknya adalah terkait dengan keanggotaan PBB di Manokwari Selatan, Papua Barat, yang menjadi batu sandungan pihaknya dalam verifikasi parpol. Afriansyah pun mengaku optimis gugatannya akan diterima oleh Bawalsu.Terpisah, Ketua Umum PKPI Abdullah Mahmud Hendropriyono mengatakan pihaknya telah menggugat KPU ke Bawaslu, pada Rabu (14/2), dan telah mendapat tanda terima pendaftaran dengan nomor 009/PS.PNM/II/2018.Menurutnya, PKPI menemukan beberapa indikasi KPU di daerah tidak profesional dalam melalukan verifikasi. Tujuan gugatan itu dilayangkan sebelum KPU mengumumkan hasil verifikasinya adalah agar KPU tidak mengambil keputusan yang salah.\"PKPI menghargai kerja keras KPU sebagai penyelenggara Pemilu 2019. Namun sayangnya kerja keras itu didak dibarengi oleh kerja profesional aparatur KPUD di beberapa daerah, sehingga merugikan PKPI,\" ujar dia, dalam keterangan tertulis yang diterima CNNIndonesia.com, Sabtu (17/2).Dia mengaku bahwa kepastian menjadi peserta Pemilu 2019 sangat penting bagi PKPI yang memiliki jutaan pendukung dan simpatisan.\"Kami memiliki lebih dari 400 anggota DPRD yang tersebar hampir di semua provinsi, kabupaten, dan kota di seluruh Indonesia,\" tutur Hendropriyono, yang juga bekas Kepala BIN itu.Sebelumnya, Hasil verifikasi KPU menyatakan PBB dan PKPI tidak memenuhi syarat untuk mengikuti Pemilu 2019. Sebab, ada sejumlah masalah syarat keanggotaan dan kepengurusan parpol di beberapa daerah.', 'inputs': array([ 4509,    50,   724,    14,  6954,   100,    15,     7,     7,\n",
      "        5666,  4317,  3240,    15,     4,  7704,     5,    16,  5666,\n",
      "        4296,    16,  1110,   100,    15,     4,   473, 19459,     5,\n",
      "          38,  8200,  8840,  5920,  1339,    15,     4, 27921,     5,\n",
      "          30,  2409, 19050,  5920,  1339,    15,     4,   227,    13,\n",
      "        5835,  1088,     5,   877,   379,    18,  1711,  6054,    54,\n",
      "          19,   759, 24305,  7047,   222,  1575,  9292,   423,     3,\n",
      "           6,  2233,  1869,    13,   328,    38,  2935,    15,     4,\n",
      "       16518,    36,     5,    16,   374,    78,    20,    38,  1202,\n",
      "         328,    15,    13, 10490,    14,   877,   759,    36,     7,\n",
      "       25260,    36,    18, 11482,    20,   353,  1910,    14,     6,\n",
      "        2465, 16124,  9468,  3013,  8086,  7031,  9644,  2110,    14,\n",
      "       12339,  1841,  2326, 30209,  8590,  3657, 10624,   604,  2250,\n",
      "        3376,  7829,  5666,  3192,  5837, 14695,  9292,  2657,    18,\n",
      "         724,    14,  2348,    15,     4,  1157, 13736,     5,     3,\n",
      "         529,    13,   796,  8908, 27946,    17,    38, 13949,    32,\n",
      "         930,    35,  2807,    22, 11889,  3013,    18, 31045,   358,\n",
      "          14,  2461,   422,    14,    17,    56,  1267, 17665,  5970,\n",
      "         930,    19, 24305,  5429,  5001,     3,  8086,  7031,  9644,\n",
      "         393,  2388,  9488, 27946,    26,    38,   964,    32,   244,\n",
      "        6429,  1605,     3,  6594, 21352,    14,   202,  1339,   285,\n",
      "       19459,   941,  3708,   951, 16069,  7355, 21561,   796,   930,\n",
      "          33,  8200, 11482,    30, 20384,    14,    23,  2272,    15,\n",
      "           4,   570, 13736,     5,    14,    16,    33,   231,  1678,\n",
      "        1238,  2372,    22,  9079,   642,  3721,   214,  5044,     3,\n",
      "       27217,   214,  8071, 11949,     3, 24358,    26,    14,   285,\n",
      "       19459,  5113,   103,    15, 16609, 11482,    18,   136,    29,\n",
      "        1789,    19,  3722,  5559, 24305,     3,  6860, 27946,    24,\n",
      "          18,  9463,    54,   137, 11482,  1152,   306, 24305,    26,\n",
      "          35,   480, 11482,    29,   187,   378,    17,   201,     3,\n",
      "           6,   473, 19459,  4412,   290,  1978, 11482,    41, 19254,\n",
      "        9292,   423,     3,   428,  4978,    26,   290,  1978,    24,\n",
      "          18,  3934,    18,  1909,  2014,    52,    32,   290,  1789,\n",
      "       13869,   879, 11482,   513,    18,   103,   136,    14,   135,\n",
      "        9454,   285, 19459,    14,     6,  2465,    18,    13,    14,\n",
      "          19,  2290,  9685,    17,   964, 25364,     3,  1065,    14,\n",
      "        2348,    15,     4,  1157, 13736,     5,     3,  9569,  2388,\n",
      "         883, 18662,    56,  1575,  9292,   423,   353,   319,    43,\n",
      "         285, 19459,    17,   164, 11378, 10586,    16, 11857,  1279,\n",
      "           3,     6,  1597,   164,    51,    37,  5677,   381, 16941,\n",
      "          17,  4136,   670,    18,   125,  2159,    14,  4799,    14,\n",
      "          16,   915,    18,   291,   100,    14,     6, 10312,   951,\n",
      "       16069,  7355, 21561,    14,    17,    42,   516,  3713,  1040,\n",
      "          24,     3,  9184,    26,    14,  2250, 24305, 11482,   249,\n",
      "        3013,    16,   285, 19459,    29,  1239,  1281,    21,  1841,\n",
      "        9292,   423,     3,  3354,    14,    64,  1274,   248,  1281,\n",
      "       11889,    16,    30,  3095, 21898,  5429,  5001,    18,   103,\n",
      "         136,     3,     1]), 'targets_plaintext': b'Tak Lolos Verifikasi, PBB dan PKPI Gugat KPU ke Bawaslu', 'targets': array([ 1486,  2256,  6054,  3376,  7829,    14,  3013,    16,   285,\n",
      "       19459,  3259,  5329, 11482,    30, 20384,     1])}\n",
      "{'inputs_plaintext': b'tajuk: KUALA LUMPUR: Kerajaan diminta terus membantu golongan petani, nelayan, peladang dan pekebun kecil susulan pemotongan insentif terbabit menerusi Belanjawan 2019. Datuk Seri Anwar Ibrahim (PH PKR-Port Dickson) berkata, sebahagian besar kelompok masyarakat itu adalah kalangan miskin yang memerlukan bantuan kerajaan demi kelangsungan hidup. \\xe2\\x80\\x9cSaya harap bantuan musim tengkujuh kepada pekebun kecil dan penoreh getah, subsidi baja padi, insentif pengeluaran padi diberikan. Insentif bersasar nelayan perlu dipastikan sampai ke kumpulan sasar. \\xe2\\x80\\x9cBegitu juga subsidi baja padi huma di Sarawak dan Sabah. Sudahlah mereka miskin, jumlah diperuntukkan pun cuma RM70 juta. \\xe2\\x80\\x9cElaun sara hidup nelayan darat yang dulunya diperuntukkan RM12.5 juta, kini dipotong kepada RM3.5 juta,\\xe2\\x80\\x9d katanya ketika membahaskan Rang Undang-Undang Perbekalan 2019 di Dewan Rakyat, hari ini. Bagaimanapun, Anwar mengakui memahami rasional di sebalik pengurangan bantuan kepada kumpulan sasar berkenaan yang bertujuan menangani masalah agihan bantuan terdahulu. Katanya, langkah perit itu perlu diambil bagi mengelak berlaku ketirisan dan penyalahgunaan kuasa, selain memastikan agihan kepada kelompok sasaran dilaksanakan secara tuntas. Kenyataan itu mendorong Menteri Kewangan, Lim Guan Eng, meminta laluan untuk mencelah dan berkata: \\xe2\\x80\\x9cIni bukan Ahli Parlimen biasa.\\xe2\\x80\\x9d Beliau kemudian mengambil sedikit masa memberi jaminan komitmen kerajaan tidak akan mengabaikan petani, nelayan, peladang dan pekebun kecil tulen. \\xe2\\x80\\x9cKita akan memastikan ketirisan yang cuba diatasi tidak akan menjadikan mereka mangsa,\\xe2\\x80\\x9d katanya. Bukan menjadi kelaziman bagi menteri mencelah ketika sesi perbahasan dan tindakan Guan Eng itu membuatkan beberapa Ahli Parlimen bangun menyuarakan bantahan, termasuk menyatakan beliau melanggar peraturan Dewan. Tan Sri Annuar Musa (BN UMNO-Ketereh) menyifatkan Guan Eng mengganggu masa wakil rakyat manakala Datuk Takiyuddin Hassan (PAS-Kota Bharu) menegaskan menteri mempunyai peluang ketika sesi penggulungan minggu depan. \\xe2\\x80\\x9cJawablah masa penggulungan nantilah,\\xe2\\x80\\x9d celah Datuk Seri Tajuddin Abdul Rahman (BN UMNO-Pasir Salak) manakala beberapa Ahli Parlimen pembangkang lain turut membantah. Speaker Datuk Mohamad Ariff Md Yusof kemudian menutup mikrofon semua Ahli Parlimen terbabit dan memberi peringatan supaya mereka menghormati pembahas sebelum memberi masa tambahan kepada Anwar menyambung perbahasannya.', 'inputs': array([ 4509,    50,   668,   767,    50,    87,  2538,   224,   272,\n",
      "         425,  1995,    14,  1415,    14, 15205,    16,  3527,   345,\n",
      "        1692, 12292,  3033,  2085,   474,  3922,   423,     3,   106,\n",
      "         188,  1056,   896,    15,     4,  1499,  1017,     7, 19379,\n",
      "        5501,     5,    60,    14,   596,   146,  1684,   158,    24,\n",
      "          35,   457,  2435,    17,   858,   521,    72,  1130,  8494,\n",
      "         312,     3,    15,     2,   650,  3856,   521,   556, 18414,\n",
      "          28,  3527,   345,    16, 16074,  2123,    14,  1792,  5966,\n",
      "        1192,    14,  3033,  1104,  1192,   502,     3, 10444, 10791,\n",
      "        1415,   101,  8539,  1052,    30,   279,  7100,     3,    15,\n",
      "           2, 17674,    42,  1792,  5966,  1192,    15,  8235,    13,\n",
      "          18,   467,    16,   343,     3,  5344,   289,    45,  2435,\n",
      "          14,   151,  1917,   393,  3174,    76,  2898,   123,     3,\n",
      "          15,     2, 10542,    13,   676,  2482,   312,  1415,  3318,\n",
      "          17,  1953,    26,  1917,    76,  1194,     3,   234,   123,\n",
      "          14,   210,  9761,    28,    76,   205,     3,   234,   123,\n",
      "          14,     2,    70,   110, 13898,  3969,  1550,     7,  2771,\n",
      "       25214,   423,    18,   373,   523,    14,    78,    20,     3,\n",
      "        1010,    14,  1056,  2117,  1859,  6661,    18,  5592,  4010,\n",
      "         521,    28,   279,  7100,   159,    17,  1106,   943,   248,\n",
      "        6829,   521,  2170,     3,   551,    14,   451,   519,  1038,\n",
      "          24,   101,   597,    43,  2720,   216,  8385,    16,  4567,\n",
      "         284,    14,   486,   310,  6829,    28,  1684,  1553,   453,\n",
      "          75, 19453,     3,  5641,    24,  2749,    82,  1004,    14,\n",
      "        1159,  1560,  1387,    14,   928,  1186,    21, 21241,    16,\n",
      "          60,    50,    15,     2,  1981,   149,   367,   333,   634,\n",
      "           3,     2,    99,   346,   187,  1098,    97,   150,  2053,\n",
      "        2279,    72,    29,    38,  8272,  1995,    14,  1415,    14,\n",
      "       15205,    16,  3527,   345,  8424,     3,    15,     2,   978,\n",
      "          38,   310,  8385,    17,  1006,  8030,    29,    38,   475,\n",
      "          45,   701,    14,     2,    70,     3,  2436,    56, 21757,\n",
      "          43,   801, 21241,   110,  1333,  5890,    16,   261,  1560,\n",
      "        1387,    24,  3964,   103,   367,   333,  7909,  5455,  4096,\n",
      "          14,   120,   249,    71,  2714,  1448,   373,     3,   543,\n",
      "         462,  9924,  3142,    15,     4,  1670,   400,     7,  1756,\n",
      "         778,  1899,     5,  3210,  1560,  1387,  4268,    97,  1248,\n",
      "         129,   445,   106, 13064,  1880,    15,     4,  7800,     7,\n",
      "       15338,  2844,     5,  1586,   801,    85,   678,   110,  1333,\n",
      "       20661,  1096,   632,     3,    15,     2, 19565,   443,   289,\n",
      "          97, 20661,  1992,   289,    14,     2, 17356,   106,   188,\n",
      "       11974,   288,  1299,    15,     4,  1670,   400,     7, 28653,\n",
      "       13845,     5,   445,   103,   367,   333,   962,    74,   134,\n",
      "        4723,     3,  6740,   106,   477,  8093,  2677,  2281,   346,\n",
      "        3493, 30023,   125,   367,   333,  2085,    16,   150,  5213,\n",
      "         318,    45,  3234,  4216,  7035,   137,   150,    97,   946,\n",
      "          28,  1056,  5896,  5890,    26,     3,     1]), 'targets_plaintext': b\"'Bantulah petani, nelayan...'\", 'targets': array([  15,   12,  227,   13, 9765,  289, 1995,   14, 1415,    3,    3,\n",
      "          3,   12,    1])}\n",
      "{'inputs_plaintext': b'tajuk: : Dianggarkan secara purata, 4,000 remaja perempuan hamil bawah 18 tahun dilaporkan setiap tahun berdasarkan statistik Kementerian Kesihatan Malaysia.Memetik statistik berkenaan, Timbalan Menteri Pembangunan Wanita, Keluarga dan Masyarakat, Hannah Yeoh berkata dari tahun 2012 hingga 2016, sebanyak 79,302 kes remaja perempuan hamil bawah 18 tahun telah dilaporkan.\\xe2\\x80\\x9cKajian Penduduk dan Keluarga Malaysia 5 yang dijalankan oleh LPPKN (Lembaga Penduduk dan Pembangunan Keluarga Negara) pada tahun 2015 turut mendapati pengetahuan mengenai aspek sesksualiti dan kesihatan reproduktif masih lagi rendah dalam kalangan remaja di Malaysia.\\xe2\\x80\\x9cBerdasarkan kajian National Health and Morbidity Survey 2017 oleh KKM, dalam kalangan remaja pernah mengadakan hubungan seks, hanya 10 peratus menyatakan pernah menggunakan kaedah kontraseptif bagi mencegah kehamilan,\\xe2\\x80\\x9d katanya di Dewan Rakyat di sini hari ini.Beliau berkata demikian sebagai menjawab soalan Kasthuriraani Patto (PH-Batu Kawan) yang meminta Kerajaan menyatakan bilangan kes bayi yang dibuang selain rancangan daripada Kerajaan.Tambah Hannah, memetik statistik Polis Diraja Malaysia (PDRM), sejumlah 911 bayi dibuang dari tahun 2010 hingga Ogos 2018, di mana 585 daripada jumlah bayi dijumpai tidak bernyawa.Justeru, katanya fokus utama Kerajaan adalah untuk memastikan keselamatan, kesihatan, kebajikan bayi dan ibunya terbela di mana ia turut menjadi tanggungjawab semua pihak termasuk ibu bapa, sekolah, komuniti, institusi keagamaan, media, pihak berkuasa negeri dan persekutuan.Malah, Kementerian juga akan memperkasa pendekatan sedia ada melalui inisiatif relevan yang mendekati golongan remaja melalui media sosial yang lebih selesa untuk mereka mendapatkan maklumat bimbingan dan nasihat.', 'inputs': array([ 4509,    50,    80, 18915,    75,  1318,    14, 11238,  1610,\n",
      "         527,  8175,   108,   390,    31,  1111,   184,    31,   468,\n",
      "        3315,   165,  1067,    40,     3, 14530, 11275,  3315,   159,\n",
      "          14,   552,    82,   774,  1173,    14,  2186,    16,  1489,\n",
      "          14, 12342, 11689,    60,    37,    31,  1058,   154,  3017,\n",
      "         147,  7667,    14,   938,   143,   247,  1610,   527,  8175,\n",
      "         108,   390,    31,    33,  1111,     3,     2, 22715,  3605,\n",
      "          16,  2186,    40,   241,    17,   852,    32,    15, 20770,\n",
      "          15,     4, 29753,  3605,    16,   774,  2186,   185,     5,\n",
      "          23,    31,   535,   134,  1055,  1810,   173,   863,   656,\n",
      "        3784,  1605,    13, 11611,    16,   780, 26504,   145,    95,\n",
      "         530,    19,   457,  1610,    18,    40,     3,     2, 10817,\n",
      "         577,  3388, 10701,    63,  4432,  8986,  2118, 23984,   568,\n",
      "          32,  3542,    14,    19,   457,  1610,   283,   935,   684,\n",
      "        3182,    14,   121,   250,   255,   249,   283,   175,   933,\n",
      "       12111,  8865,  2701,    43,  3808, 12047,    14,     2,    70,\n",
      "          18,   373,   523,    18,   192,    78,    20,     3, 11469,\n",
      "          60,   606,    41,  2133,  2105, 29672,  4119,   584,    15,\n",
      "           4,  1499,     7, 23596, 13337,     5,    17,   928,    87,\n",
      "         249,   848,   247,  2127,    17,  5201,   486,  1269,    47,\n",
      "          87,     3, 17696,  2102, 12342,    14,  6854,  3315,   617,\n",
      "        1178,    40,    15,     4,  4318,     5,    14,  1274,   482,\n",
      "        1011,  2127,  5201,    37,    31,   777,   154,   727,  2142,\n",
      "          18,   119,   241,  3529,    47,   151,  2127,  4214,    29,\n",
      "       29879,     3,  7068,    14,    70,  1945,   174,    87,    35,\n",
      "          21,   310,   490,    14,   780,    14,  2028,  2127,    16,\n",
      "        3199, 22473,    18,   119,    84,   134,    56,  1349,   125,\n",
      "          94,   120,   450,  1068,    14,   186,    14,   993,    14,\n",
      "         854,  4256,    14,   257,    14,    94,   998,    86,    16,\n",
      "        2835,     3,  6863,    14,   165,    42,    38,  9156,  1440,\n",
      "         757,    64,   116,  1127,  3910,    17,  6485,   425,  1610,\n",
      "         116,   257,   359,    17,    51,  3748,    21,    45,   520,\n",
      "         567,  7170,    16,  2708,     3,     1]), 'targets_plaintext': b'4,000 remaja bawah 18 tahun hamil setahun', 'targets': array([11238,  1610,   108,   390,    31,  8175,  1305,     1])}\n",
      "{'inputs_plaintext': b'tajuk: KOTA BHARU: Kerajaan Kelantan hari ini memutuskan tidak akan membuka pawagam di negeri itu dalam usaha mengekang masalah sosial. Pengerusi Jawatankuasa Kerajaan Tempatan, Perumahan dan Kesihatan negeri, Dr Izani Husin, berkata keputusan itu dibuat menerusi majlis mesyuarat kerajaan negeri bagi membincangkan isu berkenaan secara terperinci. \\xe2\\x80\\x9cHari ini majlis mesyuarat Kelantan berbincang dengan terperinci mengenai masalah pawagam. Kita tiada pawagam selama 30 tahun dan ia tiada masalah kerana banyak sumber jika mahu tengok cerita. \\xe2\\x80\\x9cKita boleh mengenakan syarat ketat kepada pawagam, namun jika kita buka ruang, penguatkuasaan terhadap pelanggaran syarat juga akan berdepan masalah dan perkara itu perlu diambil kira. \\xe2\\x80\\x9cPada masa sama, kita juga boleh kawal dalam pawagam, namun ketika anak-anak keluar dan pulang ke rumah selepas jam 12 tengah malam akan mengundang banyak masalah gejala sosial. \\xe2\\x80\\x9cSedangkan tahun ini, Menteri Besar, Datuk Ahmad Yakob meletakkan fokus untuk mengurangkan gejala sosial menyebabkan kita memutuskan tiada pawagam di seluruh Kelantan,\\xe2\\x80\\x9d katanya. Beliau berkata demikian ketika sesi penggulungan pada sidang Dewan Undangan Negeri (DUN) di sini, hari ini. Semalam, Mohd Syahbuddin Hashim (BN-Galas) mencadangkan kerajaan negeri membenarkan pembukaan pawagam patuh syariah di negeri ini selepas operasinya dihentikan kira-kira 20 tahun lalu. Katanya, setiap hujung minggu, ramai rakyat negeri ini ke Terengganu semata-mata untuk menonton wayang.', 'inputs': array([ 4509,    50,  4167,  8809,    50,    87,   585,    78,    20,\n",
      "        1609,    29,    38,  1473,  6061,    18,    86,    24,    19,\n",
      "         302,  7284,   248,   359,     3,   758,   895,    87,  2308,\n",
      "          14,  1623,    16,  1067,    86,    14,   122, 28958,  9786,\n",
      "          14,    60,   378,    24,   384,   474,   905,  1286,    72,\n",
      "          86,    43,  2059,    93,   159,    75,  2741,     3,    15,\n",
      "           2,  7446,    20,   905,  1286,   585,  3288,    22,  2741,\n",
      "         173,   248,  6061,     3,   872,   431,  6061,   331,   493,\n",
      "          31,    16,    84,   431,   248,    69,   127,   554,   268,\n",
      "         391,  7127,  1811,     3,    15,     2,   978,    66,  2953,\n",
      "        1281,  4124,    28,  6061,    14,   354,   268,    79,  6861,\n",
      "        1137,    14,  1971,   109,  5847,  1281,    42,    38,  1913,\n",
      "         248,    16,   273,    24,   101,   597,   369,     3,    15,\n",
      "           2,  3841,    97,    96,    14,    79,    42,    66,  7322,\n",
      "          19,  6061,    14,   354,   110,   176,     7,  1261,   784,\n",
      "          16,  1530,    30,   160,    98,   537,   417,   746,   647,\n",
      "          38,  5052,   127,   248,  3445,   359,     3,    15,     2,\n",
      "       30232,    31,    20,    14,    82,   494,    14,   106,   342,\n",
      "       24017,  1481,  1945,    21,   820,  3445,   359,   370,    79,\n",
      "        1609,   431,  6061,    18,   291,   585,    14,     2,    70,\n",
      "           3,    99,    60,   606,   110,  1333, 20661,    23,   908,\n",
      "         373,  1706,   156,    15,     4,  3849,     5,    18,   192,\n",
      "          14,    78,    20,     3,  4135,    14,   259,  4609,   443,\n",
      "        2599,  3846,    15,     4,  1670,     7,   638,    13,  3005,\n",
      "           5,  1769,    72,    86,  1515,  3413,  6061,  7460,  3552,\n",
      "          18,    86,    20,    98, 14402,  5504,   369,     7,   485,\n",
      "         427,    31,   148,     3,   551,    14,   184,  2410,  1096,\n",
      "          14,   316,   129,    86,    20,    30,   588,  2244,     7,\n",
      "        1936,    21,  6675,  9842,     3,     1]), 'targets_plaintext': b'Tiada pawagam di Kelantan', 'targets': array([2993, 6061,   18,  585,    1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('news_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 1024, 'targets': 1024}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stemming_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        ['gs://mesolitica-general/t5-data/stemming.tsv']\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def stemming_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['punca: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('stemming_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'stemming_dataset',\n",
    "    dataset_fn = stemming_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [stemming_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'punca: Sekolah Bangsa Lepar Sekolah Bangsa Lepar atau nama ringkas \" SK Lepar \" , rupa buah Sekolah Bangsa yang letak di Rpsb Batu angan Lepar Pan .', 'inputs': array([1761,   50,  133, 2337,  917, 2453,  133, 2337,  917, 2453,   46,\n",
      "        195, 4140,   15,    6, 3758,  917, 2453,   15,    6,   27, 6523,\n",
      "        206,  133, 2337,   17, 4201,   18, 1994,   25,  443, 1028,   15,\n",
      "       3384,  917, 2453, 2037,   15,    3,    1]), 'targets_plaintext': b'Sekolah Kebangsaan Lepar Sekolah Kebangsaan Lepar atau nama ringkasnya \"SK Lepar\", merupakan sebuah Sekolah Kebangsaan yang terletak di Rpsb Batu 8 Lepar Pekan.', 'targets': array([ 133,  191,  917, 2453,  133,  191,  917, 2453,   46,  195, 1183,\n",
      "         15,    6, 1760,  917, 2453,    6,   14,   34,   39,  133,  191,\n",
      "         17,   49,   18, 1994,   25,  443, 1028,  410,  917,    1])}\n",
      "{'inputs_plaintext': b'punca: \" Jump Next \" angan', 'inputs': array([ 1761,    50,    15,     6, 18374, 21637,    15,     6,    15,\n",
      "        3384,     1]), 'targets_plaintext': b'\"Jump Next\"!', 'targets': array([   15,     6,   749,  6440, 21637,     6,  1097,     1])}\n",
      "{'inputs_plaintext': b'punca: Port of Vancouver , yang besar dan paling pelbagai Kanada , laku lebih daripada C $75 bilion dalam dagang dengan angan lebih ekonomi lain cara tahun .', 'inputs': array([ 1761,    50,  3150,    58, 14326,    27,    17,   146,    16,\n",
      "         314,   179,  1904,    27,  2543,    51,    47,   398,  3324,\n",
      "        3300,   256,    19,  3668,    22,    15,  3384,    51,   194,\n",
      "          74,   583,    31,    15,     3,     1]), 'targets_plaintext': b'Port of Vancouver, yang terbesar dan paling pelbagai Kanada, melakukan lebih daripada C$75 bilion dalam perdagangan dengan 130 lebih ekonomi berlainan secara tahunan.', 'targets': array([ 3150,    58, 14326,    14,    17,   728,    16,   314,   179,\n",
      "        1904,    14,   341,    51,    47,   398, 19784,   234,   256,\n",
      "          19,  1051,    22,  9159,    51,   194,  3270,    75,  2988,\n",
      "           3,     1])}\n",
      "{'inputs_plaintext': b'punca: Long Yunus punya beberapa orang putera iaitu Long Muhammad ( Sultan Muhammad I ) yang mandul , Long Ismail , Long Jenal , dan Long Tan ( Tengku Temenggong Aria Pahlawan ) .', 'inputs': array([ 1761,    50,  3345,  7683,  3453,   103,    53,  5054,    92,\n",
      "        3345,   525,    15,     4,   403,   525,   166,    15,     5,\n",
      "          17,  3446,  7647,    27,  3345,   806,    27,  3345,  4939,\n",
      "         286,    27,    16,  3345,   543,    15,     4,  1771, 20015,\n",
      "        6701,    13, 11342,    15,     5,    15,     3,     1]), 'targets_plaintext': b'Long Yunus mempunyai beberapa orang putera iaitu Long Muhammad (Sultan Muhammad I) yang mandul, Long Ismail, Long Jenal, dan Long Tan (Tengku Temenggong Aria Pahlawan).', 'targets': array([3345, 7683,   85,  103,   53, 5054,   92, 3345,  525,   15,    4,\n",
      "       8701,  525,  166,    5,   17, 3446, 7647,   14, 3345,  806,   14,\n",
      "       3345, 4939,  286,   14,   16, 3345,  543,   15,    4,    1])}\n",
      "{'inputs_plaintext': b'punca: Dalam tahun yang sama juga , PETRONAS kembang usaha pelbagai .', 'inputs': array([ 1761,    50,   118,    31,    17,    96,    42,    27,  7158,\n",
      "          15, 12124,   302,   179,    15,     3,     1]), 'targets_plaintext': b'Dalam tahun yang sama juga, PETRONAS mengembangkan usaha kepelbagaiannya.', 'targets': array([ 118,   31,   17,   96,   42,   14, 7158, 3112,  302, 4300,   26,\n",
      "          3,    1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('stemming_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def synonym_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        ['gs://mesolitica-general/t5-data/synonyms.tsv']\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def synonym_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': tf.strings.join(['sinonim: ', ex['question']]),\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('synonym_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'synonym_dataset',\n",
    "    dataset_fn = synonym_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [synonym_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'sinonim: ketua gajah zat ehwal zon penubuhannya pedesaan pertumbuhan andai timbalan Ketua Menteri l bahagian ehwal agama Islam kemajuan tugas tugas perhubungan biarawati timbalan pemimpin uskup ll skim ekonomi negara pengetahuan penubuhan harta insan sains teknologi kemerosotan gelanggang kakak keluarganya keterangkuman gender sentimen selain Islam kerajaan isi perumahan pelan benteng desa semangat kediaman penyanyang ranah lebih kurang rupa perkebunan kedai asas tani yayasan keluar benteng perburuhan raya utiliti tebatan kelimpahan firma dalam pentadbiran internasional acara ehwal pengguna pembinaannya keusahawanan syarikat kawalan pusaka budaya kekacakan pemula dan Sukan unit ini diletakan di kaki persediaan kehormatan setiausaha komander keadaannya pulau pinang dan berkhidmat untuk merekodkan dan mereka segala berjalan yang memberitahu dengan keributan konvensyen kerajaan negara', 'inputs': array([12069,    50,  1103,  5304, 11426,  5866,  3578,  7770, 30855,\n",
      "        1023,    63,    13,    52,  5751,   202,    82,  1626,   334,\n",
      "        5866,   271,    83,  2481,  1278,  1278,  4967,  8304,    13,\n",
      "        5981,  5751,   603, 27117,    15,  2039,  2336,   194,    48,\n",
      "        1810,  1740,  1259,  3418,  1737,   626,  9208,  7651,  9446,\n",
      "        3417,    30, 20261,   667,   463,  7221,  3963,   486,    83,\n",
      "          72,  1505,  1543,  2166, 10249,   338,  1647,  2076,  6460,\n",
      "          26,  1093, 26344,    51,   470,  6523, 10772,  1796,   898,\n",
      "        7949, 14721,   784, 10249, 31148,   305, 10067, 21640,    30,\n",
      "       13894,    36,  5912,    19,   692,  5960,  1359,  5866,   635,\n",
      "       16675,  4713,   171,  1358,  6966,   865, 10044,    13,  1203,\n",
      "          54, 16400,    16,   982,   817,    20, 22032,    36,    18,\n",
      "        1309,  4781, 13353,  8305, 10934,   449,    26,  1313, 29176,\n",
      "          16,  1589,    21,  6605,    16,    45,  1073,  1804,    17,\n",
      "        1951,    22,    30, 16092,    36, 10531,    72,    48,     1]), 'targets_plaintext': b'komandan uskup topik ehwal kawasan yayasan kawasan kenaikan contoh timbalan pengerusi uskup l bahan ehwal keagamaan muslim pembinaan perusahaan korelasi komuniti timbalan pengerusi gajah ll rancangan jimat pedesaan pengajaran yayasan aset organisma sains teknologi perubahan kawasan kak pembantu rumah keterangkuman gender kepercayaan selain muslim kekaisaran domestik perumahan rancangan kota kampung kebaikan hati komuniti penyanyang dunia hampir bentuknya perladangan perniagaannya fundamental tani penubuhan di luar kota kerja-kerja raya utiliti tebatan limpahan korporat sebelah dalam kerajaan transnasional perkara ehwal pemakainya eksploitasi keusahawanan organisasi litar turunan peradaban keindahan pemuda dan sport satuan ini diletakan di bawah bawah pengurusan kewibawaan setiausaha pemerintah daerah pulau pinang dan menjamu untuk merekodkan dan menyusun segala-galanya kerja yang berhubungan dengan parti perhimpunan pentadbiran republik', 'targets': array([30253, 27117,  6049,  5866,    61, 14721,    61,  1695,  1033,\n",
      "        5751,  4028, 27117,  1626,   547,  5866,  4256,  4933,   675,\n",
      "        1943, 31146,   993,  5751,  4028,  5304,    15,  2039,  1269,\n",
      "       26083, 30855,  2623, 14721,     1])}\n",
      "{'inputs_plaintext': b'sinonim: bernas kuasanya budak mahal yang menurunkannya ayahandanya raja-raja topik yang berpaling yang mengasaskan mesir dan cinta kepada tuhannya berperang musuhnya yang berkumpul ketamadunan menarik ranah busa keraian trio perpuluhan istilah bahkan seragam ptah yang Hebat yang diberi kebahagiaan oleh ra datu yang bodoh bagi kedaulatan kawasan menyatukan dan dasar penubuhan dewi dewi dewi philopator yang diakui oleh ptah yang dimenangkan oleh ra imej mencairkan amun kecil ra ptolemy bermain-main senantiasa dikasihi ptah mendalam alas sembilan apabila aetos anak kepada aetos pinjam pendeta alexandria dan tuhan kehebatan soter dan dewi dewi Tuhan adelphoi dan dewi dewi euergetai dan dewi penaikan philopator dan dewa epiphanes eucharitos pyrrha ranting wanita wanita philinos pusing anthloporos berenike euergetis areia keturunan wanita diogenes berdiri kanephoros arsinoe philadelphos irene keturunan wanita ptolemy pastor arsinoe philopator bulan empat xandikos menyalin keturunan mesir mekhir ke', 'inputs': array([12069,    50,   181,  3377, 14483,  7719,  4116,    17,  3596,\n",
      "          26, 19684,  1428,     7,  2743,  6049,    17, 19381,    17,\n",
      "        8796,   914,  4952,    16,  2919,    28, 12182,    26,  7585,\n",
      "        2514,    26,    17,  6305, 24842,   717, 26344,  9881,    13,\n",
      "       11704,  1379,  6020,   161, 24230,  1764,  1993,  7376,  1393,\n",
      "        3536,    17,   951,  1750,    17,   608, 12769,    32,  2784,\n",
      "          13,  3476,  1420,    17, 15945,    43,  3915,    61,  6981,\n",
      "          16,   615,  1740, 15092, 15092, 15092,    15, 27551,  3109,\n",
      "        5107,    17,  6046,    32,  1393,  3536,    17,    18, 10127,\n",
      "          54,    32,  2784,    13,  2800, 31818,    15,    13,  3633,\n",
      "         345,  2784,    13,    15,  6091,  7742,  2646,   798,     7,\n",
      "        7071, 23687, 27522,  1393,  3536,  3900,   115,    13,    25,\n",
      "        2129,   169,    15,    13,   679,  1584,   176,    28,    15,\n",
      "          13,   679,  1584, 24051, 20542,   115,  2990,    13, 20619,\n",
      "          13,    16, 12182, 14203,  1630,   778,    16, 15092, 15092,\n",
      "        2649,    15,    13,  5542, 12022,    52,    16, 15092, 15092,\n",
      "          15,  6016, 24008,  3162,    16, 15092,  1190,    13,  3130,\n",
      "          15, 27551,  3109,  5107,    16,  5944, 14714, 21270,   329,\n",
      "         867,  7706,  2205,  9611,  1393,  5254, 17015,    13, 27357,\n",
      "         233,   233,    15, 27551,  3882,    25, 28006,    15,    36,\n",
      "         666, 12488,  9664,    25,   181,  7926,   660,    15,  6016,\n",
      "       24008,  2598,   669,    52,    13,  2554,   233,    18, 10145,\n",
      "         329,  3792,    15,    54,    77, 24201,  1584,  6490,  2694,\n",
      "        9532,    15, 27551,    13,  5542, 12022,    25,    15,   868,\n",
      "        7544,  2554,   233,    15,  6091,  7742,  2646,  7297,   716,\n",
      "        6490,  2694,  9532,    15, 27551,  3109,  5107,   235,   415,\n",
      "        4800,    13,  7505,  7533, 29017,  2554,   914,  4952,   914,\n",
      "         139,  3290,    30,     1]), 'targets_plaintext': b'tepu daya orang banyak yang mewarisi ayahandanya datu subjek yang paling yang mengasaskan mesir dan mencintai kepada tuhannya beradu musuhnya yang membetulkan ketamadunan membawa kekaisaran kepalanya keraian tiga puluh masa sebaliknya sama ptah yang cerah yang diberi kemakmuran oleh ra raja-raja yang tidak masuk akal bagi pemerintahan negeri berpadu dan bawah asal betari dewa philopator yang diakui oleh ptah yang dimenangkan oleh ra paradigma mengalir amun anak-anak ra ptolemy berpura-pura selamanya dikasihi ptah mahir radikal kesembilan apabila aetos kecil kepada aetos meminjamkan paderi alexandria dan dewi keagungan soter dan betari dewi adelphoi dan tuhan Tuhan euergetai dan tuhan pengangkatan philopator dan dewi epiphanes eucharitos pyrrha anak-anak kak philinos menggerakkan anthloporos berenike euergetis areia anak perempuan diogenes menjadi kanephoros arsinoe philadelphos irene nenek moyang perempuan ptolemy sami arsinoe philopator bulan berempat xandikos meniru anak-anak mesir mekhir ke', 'targets': array([20507,  1153,    53,   127,    17,  8921, 19684,  3476,  1420,\n",
      "        3871,    17,   314,    17,  8796,   914,  4952,    16, 13065,\n",
      "          28, 12182,    26,   181,    13,  1263,  2514,    26,    17,\n",
      "       15596, 24842,   368,    30,     1])}\n",
      "{'inputs_plaintext': b'sinonim: bincangkan benda pembantu allahumma ya ghaffur benar engkau maha belas kasihan ampunilah kesilapan hambamu yang rombongan dan aneka macam-macam kesilapan ini samada yang kami sukarela kerana kalahnya kami dengan hasrat kami atau yang bukan berhati-hati kerana kesilapan kami kejahatan yang kami menceritakan pada kognisi yang engkau kurniakan maupun yang kami tak ketahui kerana kejahilan kami pelanggaran yang kami mengulangi kerana tabiat degilnya kami kejahatan yang heterogen keragaman yang tak sanggup kami bilang kesilapan suntuk ahli anatomi ini pelanggaran terhadap ibubapa kami isteri kecil bayinya adik beradik konco mara setangga jirannya anak anjing ranting anak yatim papa papa budak organisma perih yang kami membelakangkan menyelitkan hasilnya pemaaf domestik nyawa merancang untuk membiarkan berlaku kami dan basuh kami dari kesilapan agnostik yang mana kami menyekutukanmu ya allah dengan haiwan tanpa kami insaf sehingga kami akibatnya besar menahan ini momen menggendong sifat baik sedangkan engkaulah yang lakukannya sifat baik itu berjalan kami kami menyekutukanmu tanpa berhati-hati dengan haiwan dekatnya kami kerana kami sangkakan merekalah yang mengagumkan mengeluarkan aneka komposisi kecemerlangan kognisi dan aneka kemegahan lagipun sehingga kami kagumi terurus sedangkan itu semuanya kehebatanmu betul betul engkau yang kompeten dikagumi kami sebaliknya selalu menyekutukanmu dengan patung kerana beranggapan merancang sanggup menawari mudharat sedangkan yang benarkan mudharat itu cuma engkau ya dharru maka bersihkanlah segala-galanya tompok kesilapan kejahatan kami agar benda ini selia', 'inputs': array([12069,    50,  5889,    54,  2908,  3659,  1611,    13,  8235,\n",
      "         162,    13,  5662,  2429, 12519,  7396,  1076, 12270, 12450,\n",
      "        2961, 18397,    15,    13,  7879,  5697,  5114,  4115,  1374,\n",
      "          17,  9974,    16, 21375,  3306,     7,  9343,  5114,    20,\n",
      "        8878,    17,   328,  6341,    69,  2680,    26,   328,    22,\n",
      "        2667,   328,    46,    17,   149,  4847,     7,  2830,    69,\n",
      "        5114,   328,  9932,    17,   328,  5508,    23,  3157, 11976,\n",
      "         408,    17, 12270, 26583,    54,  3572,    17,   328,   379,\n",
      "       17535,    69, 26553,   328,  5847,    17,   328,  7656,    69,\n",
      "       10603,   444,  5775,    26,   328,  9932,    17,    15, 24299,\n",
      "        1744, 29097,    17,   379,  3709,   328, 14913,  5114,    15,\n",
      "          25,  5514,   221, 23896,    20,  5847,   109, 17028,   328,\n",
      "        1518,   345,  2127,    26,  2811,  6919,  3965,  1618,  4510,\n",
      "         656, 20917,  3762,    26,   176,  1884, 27357,   176,  8717,\n",
      "          15, 14926,    13,    15, 14926,    13,  7719, 10682,   519,\n",
      "        2943,    17,   328, 28569,  4149,  3828,    54,  8760,  1923,\n",
      "          13,    13,   492,  2902,  3463,  2706,    21,  7290,   216,\n",
      "         328,    16, 27003,   328,    37,  5114,    15,    13, 12074,\n",
      "       10907,    17,   119,   328,  4149,   667, 20706,  1374,  5662,\n",
      "         115,   289,    22,  1179,   299,   328, 28377,   135,   328,\n",
      "         722,    26,   146,  2982,    20, 16371,   955,  1744,  3275,\n",
      "        2087,   189,  2195, 12270,   289,    17,  2935,    26,  2087,\n",
      "         189,    24,  1804,   328,   328,  4149,   667, 20706,  1374,\n",
      "         299,  4847,     7,  2830,    22,  1179,  1551,    26,   328,\n",
      "          69,   328, 10938,    54,    45,   289,    17, 18534,   966,\n",
      "       21375,  9051,  7235,  3157, 11976,   408,    16, 21375,    30,\n",
      "       18399,  1064,    95,  2498,   135,   328, 15256,    52, 22769,\n",
      "        2195,    24,  3434, 14203,  1374,  1905,  1905, 12270,    17,\n",
      "       22386, 30364,   328,     1]), 'targets_plaintext': b'bicara hati hamba allahumma ya ghaffur sesungguhnya engkau maha pengampun ampunilah dosa hambamu yang banyak dan pelbagai jenis dosa ini samada yang kami sengaja kerana kalahnya kami dengan nafsu kami atau yang tidak sengaja kerana kelalaian kami dosa yang kami tahu atas ilmu yang engkau kurniakan maupun yang kami tak ketahui kerana kejahilan kami dosa yang kami ulangi kerana sifat degilnya kami dosa yang pelbagai jenis yang tak mampu kami bilang dosa seluruh anggota tubuh ini dosa terhadap ibubapa kami suami anak anak adik beradik saudara mara jiran tetangga anak anak yatim fakir miskin orang orang sakit yang kami abaikan masukkan rasa pemaaf dalam jiwa mereka untuk memaafkan kami dan bersihkan kami dari dosa syirik yang mana kami menyekutukanmu ya allah dengan makhluk tanpa kami sedar sehingga kami rasa hebat diri ini ketika melakukan kebaikan sedangkan engkaulah yang melakukan kebaikan itu melalui kami kami menyekutukanmu tanpa sedar dengan makhluk sekeliling kami kerana kami sangkakan merekalah yang hebat mencipta pelbagai ciptaan kehebatan ilmu dan pelbagai kehebatan lagi sehingga kami kagumi mereka sedangkan itu semua kehebatanmu hanya engkau yang layak dikagumi kami juga sering menyekutukanmu dengan makhluk kerana yakin mereka boleh memberi mudharat sedangkan yang memberi mudharat itu hanya engkau ya dharru maka bersihkanlah segala kotoran dosa dosa kami agar hati ini bersih', 'targets': array([ 4853,   897,  4115,  1611,    13,  8235,   162,    13,  5662,\n",
      "        2429, 12519,  7396,  7457, 12270, 12450,  1228,    13,  7879,\n",
      "          15,    13,  7879,  5697,  7761,  4115,  1374,    17,   127,\n",
      "          16,   179,   496,  7761,     1])}\n",
      "{'inputs_plaintext': b'sinonim: hasil pembahagian beri makmal sains ilmu kimia fizik biologi sebuah perpustakaan sebuah perpustakaan tingkatan keenam sebuah perbengkelan khb sebuah perbengkelan buatan buah-buahan ruang pengetua insan dan wanita sebuah ruang cikgu tingkatan keenam sebuah ruang proses spbt mazhab sebuah ruang bentuknya fakulti sebuah ruang ict keuntungan ruang ict aspura dan aspuri keuntungan kerjasama mazhab cebisan kerjasama aspura dan aspuri hasil ruang warden manusia dan wanita wanitanya hasilnya ruang bentuk aspura dan aspuri sebuah ruang himpunan sebuah ruang vvip sebuah ruang menyantap cikgu sebuah beranda kokurikulum sebuah jabatan mazhab sebuah jawatan asrama sebuah ruang peperiksaan sebuah ruang kuak kokurikulum sebuah ruang cikgu kanan jaringan pengajian sebuah ruang kebitaraan percakapan arab sebuah ruang subjek sebuah ruang pbs sebuah ruang badar cebisan bangsal penjaga sebuah ruang kaunseling sebuah ruang mengeluarkan keuntungan ruang tamu agam dan feminin sebuah domain bebola rangkaian buah buahan gelenggang sepak takraw keuntungan medan tarian tampar hasil orbit badminton sebuah domain futsal sebuah ruang pemandu kump', 'inputs': array([12069,    50,   306,  5826,  2034,  5221,  1737,  1140,  2145,\n",
      "        6245,  5928,    39,  5159,    39,  5159,  5980,  5451,    39,\n",
      "         519, 23485,  5188,    36, 10939,   443,    39,   519, 23485,\n",
      "        5188,    36,  2813,   206,     7,  4552,  1137,  9022,  3418,\n",
      "          16,   233,    39,  1137, 31192,  5980,  5451,    39,  1137,\n",
      "         413,  5576,   443,    81,  6385,    39,  1137, 16457, 15742,\n",
      "          39,  1137,  1213,  4358,  1960,  1137,  1213,  4358,   366,\n",
      "        6884,    16,   366, 12459,  1960,   639,  6385, 29562,   639,\n",
      "         366,  6884,    16,   366, 12459,   306,  1137, 29271,   433,\n",
      "          16,   233,   233,    26,  8760,  1137,   434,   366,  6884,\n",
      "          16,   366, 12459,    39,  1137,  8598,    39,  1137,  2391,\n",
      "        1535,   397,    39,  1137,  5411,    36,  5547, 31192,    39,\n",
      "         181,  4551, 12919,    39,   266,  6385,    39,   621,  6667,\n",
      "          39,  1137,  4086,    39,  1137,  4596,    13,   139, 12919,\n",
      "          39,  1137, 31192,  1628,  2550,  1346,    39,  1137,    30,\n",
      "         901,  5969,    36, 11123,    15,    13,  3981,    39,  1137,\n",
      "        3871,    39,  1137,  1393, 11404,    39,  1137, 10395,   564,\n",
      "       29562,  1057,   363,  4060,    39,  1137,  8422,    39,  1137,\n",
      "         966,  1960,  1137, 13028,    15,    13,  4477,    16,  7392,\n",
      "        1690,   225,    39,  9775, 25819,  2005,   206,   206,    36,\n",
      "        8593,    77,  8576,   714, 17598,  1960,  3714,  5417, 27592,\n",
      "         306,  8314,  8990,    39,  9775, 24679,    39,  1137,  1521,\n",
      "        2865,  6440,     1]), 'targets_plaintext': b'buah kelas buah makmal sains kimia fizik biologi sebuah perpustakaan sebuah perpustakaan tingkatan enam sebuah bengkel khb sebuah bengkel masakan buah bilik guru lelaki dan perempuan sebuah bilik guru tingkatan enam sebuah bilik operasi spbt sekolah sebuah bilik kesihatan sekolah sebuah bilik ict buah bilik ict aspura dan aspuri buah koperasi sekolah buah koperasi aspura dan aspuri buah bilik warden lelaki dan perempuan buah bilik kesihatan aspura dan aspuri sebuah bilik mesyuarat sebuah bilik vvip sebuah bilik makan guru sebuah galeri kokurikulum sebuah pejabat sekolah sebuah pejabat asrama sebuah bilik peperiksaan sebuah bilik gerakan kokurikulum sebuah bilik guru kanan mata pelajaran sebuah bilik kebitaraan bahasa arab sebuah bilik disiplin sebuah bilik pbs sebuah bilik badar buah pondok pengawal sebuah bilik kaunseling sebuah bilik cetak buah bilik tetamu lelaki dan perempuan sebuah gelanggang bola jaring buah gelenggang sepak takraw buah gelanggang bola tampar buah gelanggang badminton sebuah gelanggang futsal sebuah bilik pengakap kump', 'targets': array([  206,  1084,   206,  5221,  1737,  2145,  6245,  5928,    39,\n",
      "        5159,    39,  5159,  5980,   810,    39,  6936, 10939,   443,\n",
      "          39,  6936,  3909,   206,  1455,   469,   215,    16,   527,\n",
      "          39,  1455,   469,  5980,     1])}\n",
      "{'inputs_plaintext': b'sinonim: cukup pemakanan sanubari itali alternatif yang basi kognisi sains dan teknologi sedunia memetakan giovanni domenico cassini berkesan ilmu falak yang membentuk kuantiti pemecahan dekan menjawab pertubuhan suria alessandro volta pintar fizik yang menandakan pengumpul elektrik penasihat layak matematik yang bercakap kenaikan hebat sebelah dalam lega mengeluarkan iaitu lagrange fibonacci dan gerolamo cardano yang karib kerana ars magna yang diiktiraf bersahaja mengagihkan pengawasan muda yang tertua pukul matematik marcello malpighi kepintaran ahli kimia dan pengasasanatomi mikro lazzaro spallanzani bijak biologi yang menyempurnakan penyiasatan keras bertindak balas janji pertubuhan pembelaan ayam itik dan andaian kader camillo golgi anggota fizik patologi dan penakluk sifat baik nobel yang popular kerana merasakan perang golgi menyelidik mendapat rujukan kepada dorongan imannya neuron dan guglielmo marconi yang ikuti paparan nobel domestik fizik kerana bercetak radio', 'inputs': array([12069,    50,  1062,  7369,    15,  1279,   198, 12152,   655,\n",
      "          13,   440,  2834,    17,    15, 14600,  3157, 11976,   408,\n",
      "        1737,    16,   626, 17333, 10102,  5888,    15, 16610,  3126,\n",
      "         794,  1742,   991, 11773, 11355,    25,  2689,  1827,  1140,\n",
      "       25899,    17,   721,  8218, 18305,    36,   444,    54,  2133,\n",
      "        2121, 11139,    15,    13,  5225,    13, 16069,    15, 17894,\n",
      "          13,  3490,  6245,    17,  4542,    15, 23925,  1485,  5937,\n",
      "        1400,  4896,    17,  2893,  1695,  2715,  1672,    19,  7795,\n",
      "          13,   966,    92,    15,  9706, 12497,  2338,  5029,    13,\n",
      "       19692,    16,    15,  1869,  9256,  1538, 15341,    13,   785,\n",
      "          17, 24257,    69,    15,    13,  3923,   856,    13, 12566,\n",
      "          17,  3267, 27071, 14755,  5922,   671,    17,  5530,  1762,\n",
      "        4896,  8268,  8960,   161, 12457,  2236, 15714, 25478,   221,\n",
      "        2145,    16,  4024,    36,    13, 16409,  4612,  1419,  7379,\n",
      "          13,  1146,  5576,  5802,    36,  3577,    52,  4637,  5928,\n",
      "          17,  8317,  8787,  1978,  1242,  1524,  1918,  2121,  9023,\n",
      "        2950,   655,   581,    16, 11804, 23131,    15,  5482, 14146,\n",
      "        1323,  1161,   381,  6245, 26236,    16,  1190,    13,   139,\n",
      "        4282,  2087,   189,  1511,  3352,    17,  1367,    69,  4381,\n",
      "        1018,  1323,  1161, 27458,   231,  3165,    28, 11216, 14628,\n",
      "          26, 28442,    16,    15,  1668,  8141,   844,  1538,  8268,\n",
      "        3010,    52,    17,  2966,    52,  8714,  1511,  3352,  2902,\n",
      "        6245,    69, 26274,  2515,     1]), 'targets_plaintext': b'sedang khasiat jiwa itali lain yang lama ilmu sains dan teknologi sedunia ialah giovanni domenico cassini cekap Astronomi yang membuat amaun penemuan pengetua membalas organisasi suria alessandro volta pakar fizik yang membentangkan bateri elektrik pakar mahir matematik yang bermegah pertumbuhannya besar dalam lapang mereka iaitu lagrange fibonacci dan gerolamo cardano yang akrab kerana ars magna yang diiktiraf awam bersama-sama perhatian mentah yang senior kepak matematik marcello malpighi kecerdikan doktor dan pengasasanatomi mikroskopik lazzaro spallanzani layak biologi yang menghabiskan pertanyaan nyaring tentang penugasan persatuan pembiakan ternakan dan agakan sel camillo golgi ahli fizik patologi dan pemenang merit nobel yang pop kerana membezakan kusut golgi memasuki mengawali petikan kepada pengenalan doktrin neuron dan guglielmo marconi yang berikut persembahan nobel dalam fizik kerana menerbitkan radio', 'targets': array([  352, 20550,  3528,   655,    13,   440,    74,    17,   459,\n",
      "        1140,  1737,    16,   626, 17333,    57,    15, 16610,  3126,\n",
      "         794,  1742,   991, 11773, 11355,    25,  2689,  4471, 29044,\n",
      "          17,   178, 18983,  3694,     1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('synonym_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quora_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-general/t5-data/quora-0-100k.tsv',\n",
    "            'gs://mesolitica-general/t5-data/quora-100k-200k.tsv',\n",
    "            'gs://mesolitica-general/t5-data/quora-200k-300k.tsv',\n",
    "            'gs://mesolitica-general/t5-data/quora-400k-500k.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def quora_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': ex['question'],\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('quora_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'quora_dataset',\n",
    "    dataset_fn = quora_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [quora_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'soalan1: Apa yang sepatutnya menonton rancangan TV sebelum anda mati? soalan2: Adakah ada yang mesti menonton rancangan TV?', 'inputs': array([2105,  112,   50, 1819,   17, 1593, 6675, 1269, 1414,  137,   63,\n",
      "         13, 1218,  275, 2105,  143,   50, 3450,   64,   17, 1732, 6675,\n",
      "       1269, 1414,  275,    1]), 'targets_plaintext': b'sama', 'targets': array([96,  1])}\n",
      "{'inputs_plaintext': b'soalan1: Bagaimanakah kebanyakan orang mati? soalan2: Bagaimana orang mati?', 'inputs': array([ 2105,   112,    50, 24282,   989,    53,  1218,   275,  2105,\n",
      "         143,    50,  5268,    53,  1218,   275,     1]), 'targets_plaintext': b'tak sama', 'targets': array([379,  96,   1])}\n",
      "{'inputs_plaintext': b'soalan1: Apa maksudnya jika anjing muntah buih putih? soalan2: Mengapa puppy saya muntah busa putih?', 'inputs': array([ 2105,   112,    50,  1819, 18172,   268,  1884, 18391,  5687,\n",
      "        2943,  1658,   275,  2105,   143,    50,  9675,  6697,  4739,\n",
      "         157,   113, 18391,  9881,    13,  1658,   275,     1]), 'targets_plaintext': b'sama', 'targets': array([96,  1])}\n",
      "{'inputs_plaintext': b'soalan1: Menindas saya suka saya kembali; apa yang kita buat sekarang? soalan2: Kakak saya memberitahu saya bahawa saya suka dia tanpa saya meminta dia berbuat demikian. Apa yang perlu saya lakukan sekarang?', 'inputs': array([2105,  112,   50, 1211, 8156,   25,  113, 2301,  113,  448,  111,\n",
      "        294,   17,   79,  590,  769,  275, 2105,  143,   50, 7814,   13,\n",
      "        139,  113, 1951,  113,   90,  113, 2301,   18,   13,  299,  113,\n",
      "        928,   18,   13, 3320,  606,    3, 1819,   17,  101,  113, 2935,\n",
      "        769,  275,    1]), 'targets_plaintext': b'tak sama', 'targets': array([379,  96,   1])}\n",
      "{'inputs_plaintext': b'soalan1: Bagaimana rasis terlalu rasis? soalan2: Bagaimana rasis anda?', 'inputs': array([ 2105,   112,    50,  5268, 13676,  1254, 13676,   275,  2105,\n",
      "         143,    50,  5268, 13676,    63,    13,   275,     1]), 'targets_plaintext': b'tak sama', 'targets': array([379,  96,   1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('quora_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def snli_dataset(split, shuffle_files = False):\n",
    "    del shuffle_files\n",
    "    ds = tf.data.TextLineDataset(\n",
    "        [\n",
    "            'gs://mesolitica-general/t5-data/snli-part1.tsv',\n",
    "            'gs://mesolitica-general/t5-data/snli-part2.tsv',\n",
    "            'gs://mesolitica-general/t5-data/snli-part3.tsv',\n",
    "            'gs://mesolitica-general/t5-data/snli-part4.tsv',\n",
    "            'gs://mesolitica-general/t5-data/snli-part5.tsv',\n",
    "            'gs://mesolitica-general/t5-data/snli-part6.tsv',\n",
    "            'gs://mesolitica-general/t5-data/snli-pary7.tsv',\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    ds = ds.map(\n",
    "        functools.partial(\n",
    "            tf.io.decode_csv,\n",
    "            record_defaults = ['', ''],\n",
    "            field_delim = '\\t',\n",
    "            use_quote_delim = False,\n",
    "        ),\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "    ds = ds.map(lambda *ex: dict(zip(['question', 'answer'], ex)))\n",
    "    return ds\n",
    "\n",
    "\n",
    "def snli_preprocessor(ds):\n",
    "    def to_inputs_and_targets(ex):\n",
    "        return {\n",
    "            'inputs': ex['question'],\n",
    "            'targets': ex['answer'],\n",
    "        }\n",
    "\n",
    "    return ds.map(\n",
    "        to_inputs_and_targets,\n",
    "        num_parallel_calls = tf.data.experimental.AUTOTUNE,\n",
    "    )\n",
    "\n",
    "\n",
    "t5.data.TaskRegistry.remove('snli_dataset')\n",
    "t5.data.TaskRegistry.add(\n",
    "    'snli_dataset',\n",
    "    dataset_fn = snli_dataset,\n",
    "    splits = ['train'],\n",
    "    text_preprocessor = [snli_preprocessor],\n",
    "    sentencepiece_model_path = vocab,\n",
    "    postprocess_fn = t5.data.postprocessors.lower_text,\n",
    "    metric_fns = [t5.evaluation.metrics.accuracy],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs_plaintext': b'ayat1: Orang berada di pentas panggung. ayat2: orang berada di atas pentas', 'inputs': array([1774,  112,   50,  751,  351,   18, 3555, 7934,    3, 1774,  143,\n",
      "         50,   53,  351,   18,  107, 3555,    1]), 'targets_plaintext': b'berkait', 'targets': array([3754,    1])}\n",
      "{'inputs_plaintext': b'ayat1: Seorang budak kecil telah masuk ke kabinet dan mendapat tepung dan crisco seluruh dirinya. ayat2: Anak lelaki itu sedang kacau.', 'inputs': array([ 1774,   112,    50,  1952,  7719,   345,    33,   612,    30,\n",
      "        4945,    16,   231,  7640,    16,    15, 10092,  6603,   291,\n",
      "        1476,     3,  1774,   143,    50,  2026,   215,    24,   352,\n",
      "       19112,     3,     1]), 'targets_plaintext': b'berkait', 'targets': array([3754,    1])}\n",
      "{'inputs_plaintext': b'ayat1: Seorang gadis kecil mengikuti dua lelaki dengan payung ke bawah jalan. ayat2: Gadis kecil sedang menunggang kuda.', 'inputs': array([ 1774,   112,    50,  1952,  3903,   345,  1841,   102,   215,\n",
      "          22, 10750,    30,   108,   385,     3,  1774,   143,    50,\n",
      "       21478,   345,   352, 10693,  3971,     3,     1]), 'targets_plaintext': b'percanggahan', 'targets': array([15135,     1])}\n",
      "{'inputs_plaintext': b'ayat1: Pelancong menunggu di perhentian kereta api. ayat2: Pelancong berjalan di sekitar muzium seni.', 'inputs': array([ 1774,   112,    50, 21981,  2183,    18, 19849,   542,  1007,\n",
      "           3,  1774,   143,    50, 21981,  1804,    18,   350,  8743,\n",
      "        1177,     3,     1]), 'targets_plaintext': b'percanggahan', 'targets': array([15135,     1])}\n",
      "{'inputs_plaintext': b'ayat1: Seorang wanita sedang menunggu dengan anak-anak ketika dia diperiksa di Walmart. ayat2: Seorang wanita mempunyai anak dengannya di kaunter cek.', 'inputs': array([ 1774,   112,    50,  1952,   233,   352,  2183,    22,   176,\n",
      "           7,  1261,   110,    18,    13,  8744,    18,  6096, 12701,\n",
      "           3,  1774,   143,    50,  1952,   233,    85,   176,    22,\n",
      "          26,    18,  5649,  9404,     3,     1]), 'targets_plaintext': b'berkait', 'targets': array([3754,    1])}\n"
     ]
    }
   ],
   "source": [
    "nq_task = t5.data.TaskRegistry.get('snli_dataset')\n",
    "ds = nq_task.get_dataset(\n",
    "    split = 'train', sequence_length = {'inputs': 256, 'targets': 32}\n",
    ")\n",
    "for ex in tfds.as_numpy(ds.take(5)):\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
