{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'mesolitica-storage.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def cleaning(string):\n",
    "    string = string.replace('\\n', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-general')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "from pathlib import Path\n",
    "import json\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../bert/dumping-parliament.txt',\n",
       " '../bert/dumping-iium.txt',\n",
       " '../bert/dumping-wiki.txt',\n",
       " '../bert/dumping-news.txt',\n",
       " '../bert/dumping-watpadd.txt',\n",
       " '../bert/dumping-pdf.txt']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files = glob('../bert/dumping*')\n",
    "files = [i for i in files if 'insta' not in i and 'twitter' not in i and 'combined' not in i]\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../bert/dumping-parliament.txt 890823\n",
      "../bert/dumping-iium.txt 1121978\n",
      "../bert/dumping-wiki.txt 1715551\n",
      "../bert/dumping-news.txt 1791784\n",
      "../bert/dumping-watpadd.txt 1445379\n",
      "../bert/dumping-pdf.txt 596417\n"
     ]
    }
   ],
   "source": [
    "for file in files:\n",
    "    with open(file) as fopen:\n",
    "        data = list(filter(None, fopen.read().split('\\n')))\n",
    "    print(file, len(data))\n",
    "    s = Path(file).stem\n",
    "    filename = f'{s}.tsv'\n",
    "    with tf.io.gfile.GFile(filename, 'w') as outfile:\n",
    "        for i in range(len(data)):\n",
    "            outfile.write('%s\\t%s\\n' % ('', cleaning(data[i])))\n",
    "    \n",
    "    blob = bucket.blob(f't5-data/{filename}')\n",
    "    blob.upload_from_filename(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../bert/dumping-parliament.txt 70046\n",
      "../bert/dumping-iium.txt 17837\n",
      "../bert/dumping-wiki.txt 326735\n",
      "../bert/dumping-news.txt 105469\n",
      "../bert/dumping-watpadd.txt 55091\n",
      "../bert/dumping-pdf.txt 55538\n"
     ]
    }
   ],
   "source": [
    "for file in files:\n",
    "    with open(file) as fopen:\n",
    "        data = fopen.read().split('\\n')\n",
    "    results, result = [], []\n",
    "    for i in data:\n",
    "        if not len(i) and len(result):\n",
    "            results.append('. '.join(result))\n",
    "            result = []\n",
    "        else:\n",
    "            result.append(i)\n",
    "    print(file, len(results))\n",
    "    s = Path(file).stem\n",
    "    filename = f'{s}-pair.tsv'\n",
    "    \n",
    "    with tf.io.gfile.GFile(filename, 'w') as outfile:\n",
    "        for i in range(len(results)):\n",
    "            outfile.write('%s\\t\\n' % (cleaning(results[i])))\n",
    "    \n",
    "    blob = bucket.blob(f't5-data/{filename}')\n",
    "    blob.upload_from_filename(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('stemming.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "with tf.io.gfile.GFile('stemming.tsv', \"w\") as outfile:\n",
    "    for i in range(len(data)):\n",
    "        outfile.write(\"%s\\t%s\\n\" % (cleaning(data[i][1]), cleaning(data[i][0])))\n",
    "\n",
    "blob = bucket.blob('t5-data/stemming.tsv')\n",
    "blob.upload_from_filename('stemming.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('synonyms.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "with tf.io.gfile.GFile('synonyms.tsv', \"w\") as outfile:\n",
    "    for i in range(len(data)):\n",
    "        outfile.write(\"%s\\t%s\\n\" % (cleaning(data[i][1]), cleaning(data[i][0])))\n",
    "        \n",
    "blob = bucket.blob('t5-data/synonyms.tsv')\n",
    "blob.upload_from_filename('synonyms.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../malaya-dataset/Malaya-Dataset/text-similarity/quora/0-100k.json 99993\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/quora/400k-500k.json 4290\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/quora/300k-400k.json 99995\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/quora/200k-300k.json 99992\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/quora/100k-200k.json 100000\n"
     ]
    }
   ],
   "source": [
    "files = glob('../malaya-dataset/Malaya-Dataset/text-similarity/quora/*.json')\n",
    "labels = {0: 'tak sama', 1: 'sama'}\n",
    "\n",
    "for file in files:\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    print(file, len(data))\n",
    "    s = Path(file).stem\n",
    "    filename = f'quora-{s}.tsv'\n",
    "    with tf.io.gfile.GFile(filename, 'w') as outfile:\n",
    "        for i in range(len(data)):\n",
    "            if len(data[i]) != 2:\n",
    "                continue\n",
    "            label = labels[data[i][1]]\n",
    "            splitted = data[i][0].split(' <> ')\n",
    "            if len(splitted) != 2:\n",
    "                continue\n",
    "            q = f'soalan1: {cleaning(splitted[0])} soalan2: {cleaning(splitted[1])}'\n",
    "            outfile.write('%s\\t%s\\n' % (q, label))\n",
    "    \n",
    "    blob = bucket.blob(f't5-data/{filename}')\n",
    "    blob.upload_from_filename(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/pary7.json 30934\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/part2.json 49998\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/part1.json 50000\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/part4.json 50000\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/part6.json 100000\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/part3.json 50000\n",
      "../malaya-dataset/Malaya-Dataset/text-similarity/snli/part5.json 49998\n"
     ]
    }
   ],
   "source": [
    "labels = {'contradiction': 'percanggahan', 'entailment': 'berkait'}\n",
    "\n",
    "files = glob('../malaya-dataset/Malaya-Dataset/text-similarity/snli/*.json')\n",
    "\n",
    "for file in files:\n",
    "    with open(file) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    print(file, len(data))\n",
    "    \n",
    "    s = Path(file).stem\n",
    "    filename = f'snli-{s}.tsv'\n",
    "    with tf.io.gfile.GFile(filename, 'w') as outfile:\n",
    "        for i in range(len(data)):\n",
    "            if len(data[i]) != 2:\n",
    "                continue\n",
    "\n",
    "            label = labels.get(data[i][0], data[i][0])\n",
    "            splitted = data[i][1].split(' <> ')\n",
    "            if len(splitted) != 2:\n",
    "                continue\n",
    "            q = f'ayat1: {cleaning(splitted[0])} ayat2: {(splitted[1])}'\n",
    "            outfile.write('%s\\t%s\\n' % (q, label))\n",
    "    \n",
    "    blob = bucket.blob(f't5-data/{filename}')\n",
    "    blob.upload_from_filename(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('translated-validation.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "questions, answers = [], []\n",
    "for row in data:\n",
    "    if '<>' not in row:\n",
    "        q, a = row.split('? ')\n",
    "        q = f'{q}?'\n",
    "    else:\n",
    "        q, a = row.split('<>')\n",
    "    questions.append(q.strip())\n",
    "    answers.append(a.strip())\n",
    "    \n",
    "with tf.io.gfile.GFile('qa-validation.tsv', \"w\") as outfile:\n",
    "    for i in range(len(questions)):\n",
    "        outfile.write(\"%s\\t%s\\n\" % (cleaning(questions[i]), cleaning(answers[i])))\n",
    "        \n",
    "blob = bucket.blob('t5-data/qa-validation.tsv')\n",
    "blob.upload_from_filename('qa-validation.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('translated-train.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "questions, answers = [], []\n",
    "for row in data:\n",
    "    try:\n",
    "        if '<>' not in row:\n",
    "            q, a = row.split('? ')\n",
    "            q = f'{q}?'\n",
    "        else:\n",
    "            q, a = row.split('<>')\n",
    "        questions.append(q.strip())\n",
    "        answers.append(a.strip())\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "with tf.io.gfile.GFile('qa-train.tsv', \"w\") as outfile:\n",
    "    for i in range(len(questions)):\n",
    "        outfile.write(\"%s\\t%s\\n\" % (cleaning(questions[i]), cleaning(answers[i])))\n",
    "        \n",
    "blob = bucket.blob('t5-data/qa-train.tsv')\n",
    "blob.upload_from_filename('qa-train.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !rm -rf __MACOSX/news\n",
    "# !rm -rf news\n",
    "# !wget https://huseinhouse-data.s3-ap-southeast-1.amazonaws.com/news.zip\n",
    "# !unzip news.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_news(string):\n",
    "    string = string.lower()\n",
    "    return 'javascript is disabled' in string or 'requires javascript' in string or 'javascript' in string \\\n",
    "    or 'président' in string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news = glob('news/*.json')\n",
    "\n",
    "results = []\n",
    "for n in news:\n",
    "    with open(n) as fopen:\n",
    "        data = json.load(fopen)\n",
    "    for i in data:\n",
    "        if not filter_news(i['text']) and i['language'] != 'ENGLISH' and len(i['text']) and len(i['title']):\n",
    "            results.append((cleaning(i['text']), cleaning(i['title'])))\n",
    "            \n",
    "with tf.io.gfile.GFile('news-title.tsv', \"w\") as outfile:\n",
    "    for i in range(len(results)):\n",
    "        outfile.write(\"%s\\t%s\\n\" % (results[i][0], results[i][1]))\n",
    "        \n",
    "blob = bucket.blob('t5-data/news-title.tsv')\n",
    "blob.upload_from_filename('news-title.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malaya-Dataset/master/news/news-30k/news-30k.json.zip\n",
    "# !unzip news-30k.json.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news-30k.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "results = []\n",
    "for i in data:\n",
    "    if not filter_news(i['text']) and i['language'] == 'malay' and len(i['text']) and len(i['title']):\n",
    "        results.append((cleaning(i['text']), cleaning(i['title'])))\n",
    "        \n",
    "with tf.io.gfile.GFile('news-title2.tsv', \"w\") as outfile:\n",
    "    for i in range(len(results)):\n",
    "        outfile.write(\"%s\\t%s\\n\" % (results[i][0], results[i][1]))\n",
    "        \n",
    "blob = bucket.blob('t5-data/news-title2.tsv')\n",
    "blob.upload_from_filename('news-title2.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
