{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://malaya-dataset.s3-ap-southeast-1.amazonaws.com/crawler/wattpad/wattpad.zip\n",
    "# !unzip wattpad.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4082"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "texts = glob('texts/*.txt')\n",
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'by raynanur'.find('by')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "removes = ['by', 'category', 'genre', 'language', 'status', 'published', 'updated', 'End file',\n",
    "'packaged', 'chapters', 'publisher', 'story url', 'summary', 'language', 'read count',\n",
    "          'TABLE OF', 'Author']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import cleaning\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4082/4082 [01:17<00:00, 52.34it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = []\n",
    "\n",
    "for i in tqdm(range(len(texts))):\n",
    "\n",
    "    with open(texts[i]) as fopen:\n",
    "        c = fopen.read().split('\\n')\n",
    "        \n",
    "    t, last = [], 0\n",
    "\n",
    "    i = 0\n",
    "    while i < len(c):\n",
    "        \n",
    "        text = c[i]\n",
    "        \n",
    "        if any([text.lower().find(r.lower()) == 0 for r in removes]):\n",
    "            i += 1\n",
    "            continue\n",
    "        \n",
    "        text = cleaning.cleaning(text)\n",
    "\n",
    "        if len(text) > 5:\n",
    "            if len(text.split()) > 1:\n",
    "                t.append(text)\n",
    "            last = i\n",
    "        else:\n",
    "            if len(t) and (i - last) > 2:\n",
    "                t.append('')\n",
    "                outer.extend(t)\n",
    "                t = []\n",
    "                last = i\n",
    "            elif not len(t):\n",
    "                last = i\n",
    "\n",
    "        i += 1\n",
    "    \n",
    "    if len(t):\n",
    "        t.append('')\n",
    "        outer.extend(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1500470"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(outer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 811 ms, sys: 447 ms, total: 1.26 s\n",
      "Wall time: 1.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "temp_vocab = list(set(cleaning.multiprocessing(outer, cleaning.unique_words)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70627\n",
      "CPU times: user 447 ms, sys: 160 ms, total: 607 ms\n",
      "Wall time: 1.53 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# important\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.duplicate_dots_marks_exclamations, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 269809.04it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2225\n",
      "CPU times: user 242 ms, sys: 148 ms, total: 390 ms\n",
      "Wall time: 553 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# important\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.remove_underscore, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 273974.38it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaning.isolate_spamchars([''])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CPU times: user 487 ms, sys: 204 ms, total: 691 ms\n",
      "Wall time: 1.92 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# important\n",
    "temp_dict = cleaning.multiprocessing(outer, cleaning.isolate_spamchars, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1584\n",
      "CPU times: user 192 ms, sys: 164 ms, total: 356 ms\n",
      "Wall time: 439 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.break_short_words, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 278360.10it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "520\n",
      "CPU times: user 205 ms, sys: 156 ms, total: 360 ms\n",
      "Wall time: 456 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.break_long_words, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 278492.65it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43656\n",
      "CPU times: user 351 ms, sys: 168 ms, total: 519 ms\n",
      "Wall time: 574 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.remove_ending_underscore, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 263537.67it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38988\n",
      "CPU times: user 306 ms, sys: 152 ms, total: 458 ms\n",
      "Wall time: 518 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.remove_starting_underscore, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 263058.43it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "367648\n",
      "CPU times: user 1.24 s, sys: 216 ms, total: 1.46 s\n",
      "Wall time: 1.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.end_punct, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:05<00:00, 253354.10it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130482\n",
      "CPU times: user 596 ms, sys: 161 ms, total: 756 ms\n",
      "Wall time: 881 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.start_punct, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:06<00:00, 240487.81it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1308\n",
      "CPU times: user 244 ms, sys: 153 ms, total: 397 ms\n",
      "Wall time: 559 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "temp_dict = cleaning.multiprocessing(temp_vocab, cleaning.join_dashes, list_mode = False)\n",
    "print(len(temp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1500470/1500470 [00:06<00:00, 246021.36it/s]\n"
     ]
    }
   ],
   "source": [
    "outer = cleaning.string_dict_cleaning(outer, temp_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tak ada yang lebih hina dari perlakuanmu pada kami',\n",
       " 'Kaulah punggung pemberani yang memberontak setiap hari',\n",
       " 'Hanya tangisan yang kau ukir pada hidup kami',\n",
       " 'Hanya satu yang ku pinta dan ku damba',\n",
       " 'Kau pergi atau aku yang mati',\n",
       " '',\n",
       " 'Bait demi bait kususun disertai ambisi',\n",
       " 'Getaran emosi memuncak tak bertepi',\n",
       " 'Renjana mendorong sukma menjadi gusar',\n",
       " 'Mencuar rintihan pilu pada emas tempawan',\n",
       " 'Sungkawa beradu pedar menghapus harmoni',\n",
       " 'Harmoni bahari berubah jadi duri',\n",
       " '',\n",
       " 'Jadi serbuk bak abu vulkanik',\n",
       " 'Tempo hari menyerang pribumi',\n",
       " 'Dewa amor yang kupanggil ketika rembulan tergelak',\n",
       " 'Berpijak pada dewi bahari',\n",
       " 'Menolak nestapa yang ku beri',\n",
       " 'Mengusir kejam dengan seringai berhati besi',\n",
       " '',\n",
       " 'Dewa amor',\n",
       " 'Kau adalah bagian lego diri ini',\n",
       " 'Yang berganti menjadi aksara puisi',\n",
       " 'Kini , esok dan nanti',\n",
       " 'Lampung , 28 Juli 2018',\n",
       " '',\n",
       " 'Perempuan yang Bercanda Bersama Puisi',\n",
       " '',\n",
       " 'Perihal hujan dan matahari',\n",
       " 'Menjadi saksi perihnya mencintai',\n",
       " 'Aku pernah bersembunyi di balik hujan',\n",
       " 'Atas air yang menggenang di pipi',\n",
       " '',\n",
       " 'Akupun pernah memeluk matahari',\n",
       " 'Ketika pelukmu berjalan membelakangi',\n",
       " 'Saat senja tetiba ,',\n",
       " 'Aku memilih berbalik arah',\n",
       " 'Mencari celah untuk merekah',\n",
       " '',\n",
       " 'Namun , aku lupa',\n",
       " 'Aku bukan setangkai bunga',\n",
       " 'Aku hanyalah seorang perempuan',\n",
       " 'Yang suka bercanda bersama puisi',\n",
       " '',\n",
       " 'Taman Sari , Lampung 2018',\n",
       " '',\n",
       " 'KISAH DI DALAM LIFT',\n",
       " '',\n",
       " 'DIDALAM LIFT YANG SETAM',\n",
       " '',\n",
       " 'nama saya lusi . beberapa hari yang lalu saya dan teman2 saya sempat mengalami',\n",
       " 'kisah seram di kampus . Hari itu saya dan kedua teman saya , eca dan nay',\n",
       " 'kebetulan ada jadwal tutor dengan senior sekitar jam setengah 7 malam di',\n",
       " 'lantai 3 .',\n",
       " 'Karna masih sepi dan kebetulan tutor belum datang dan juga sudah masuk waktu',\n",
       " 'maghrib , jadi kami solat maghrib terlebih dahulu di mushola lantai 4 . Setelah',\n",
       " 'selesai solat , kami balik lagi ke lantai 3 , tapi karna malas turun tangga jadi',\n",
       " 'kami memutuskan untuk menggunakan lift . Ketika kami membuka lift , memang sudah',\n",
       " 'ada orang , jumlahnya sekitar 2orang . Masing2 turun di lantai 4 dan 5 Posisi',\n",
       " 'kami setelah 2orang tersebut turun itu saya berada di pojok kanan tepat di',\n",
       " 'depan tombol lift , nay di tengah , dan eca di pojok kiri .***',\n",
       " 'Ketika orang terakhir turun di lantai 5 dan pintu lift mulai tertutup saya',\n",
       " 'langsung menekan lantai 3 , tapi tiba2 lift yang kita naiki menuju ke lantai 7 ,',\n",
       " 'berarti di lantai 7 ada orang yang ingin menggunakan lift juga . Lantai 7 ,',\n",
       " 'pintu lift pun terbuka . Saya melihat sekeliling , tidak ada seorang pun didepan',\n",
       " 'lift . Suasana pun hening karena memang jam segitu sudah tidak ada lagi kelas',\n",
       " 'yang digunakan untuk belajar . \" nay !\" panggil saya , sambil buru2 memegang',\n",
       " 'tangannya yang ingin keluar lift , mungkin kira2 sudah satu satu langkah di',\n",
       " 'depan lift , \" oh iya lantai 7 , kirain udah di lantai 3 si \". Kedua teman saya',\n",
       " 'memang dari tadi asik ngobrol jadi tidak memerhatikan . Saya pun langsung',\n",
       " 'menekan tombol untuk menutup pintu lift , namun tiba2 pintu lift tersebut',\n",
       " 'terbuka kembali seperti ada orang yang ingin masuk dengan menekan open dari',\n",
       " 'luar . Saya berusaha tenang dan mencoba untuk tidak panik . Menuju lantai 6 ,',\n",
       " 'tiba2 eca angkat bicara \" kok perasaan gue tiba2 ga en . \" \"hust . \" Kataku',\n",
       " 'memperingatkan karena pada saat itu mereka belum ada firasat apa2 . Namun',\n",
       " 'mereka berdua saling tatap2an . Pintu lift pun terbuka kembali di lantai 6 ,',\n",
       " 'namun kali ini suasana di lantai itu tidak begitu hening , saya samar2',\n",
       " 'mendengar suara bapak2 lagi ngobrol . \" untung udah ada orang \" kata eca . Saya',\n",
       " 'kira yang dimaksud eca adalah suara bapak2 yang saya dengar tadi , namun mata',\n",
       " 'eca seperti melihat ke suatu objek ketika berbicara seperti itu . Karena',\n",
       " 'penasaran , saya pun sempat melihat ke sekitar namun tidak ada satu orang pun .',\n",
       " 'Selepas kejadian di dua lantai tersebut , pintu lift selalu terbuka di setiap',\n",
       " 'lantainya . Untungnya lantai 5 dan lantai 4',\n",
       " 'masih sedikit ramai dan masih ada beberapa orang yang lalu lalang jadi kami',\n",
       " 'mulai merasa sedikit tenang . Sampai di lantai 3 , kami pun langsung saling',\n",
       " 'cerita di depan kelas . \" kenapa sih si ? Ada apa ? Kok tadi hsstt-in eca ?\"',\n",
       " '\" engga , tadi gue ngerasa gak enak aja , padahal gue udah pencet lantai 3 tiba2',\n",
       " 'lift naik ke lantai 7 , dan pas di lantai 7 pintu liftnya juga sempet kebuka',\n",
       " 'lagi . Padahal gue mencetnya close dan diluar ga ada siapa2 kan ? Jadi siapa yg',\n",
       " 'mencet open dari luar coba ? Kalo gue ceritain ,',\n",
       " 'kalian jadi takut dan nanti malah ada yg nampakin \" \" oh iya yah , untung di',\n",
       " 'lantai 6 eca sempet liat ada orang si \" \" oh bapak2 ya ? Gue juga denger tapi ga',\n",
       " 'liat ada bapak2 \" \" bukan si , ada cewe depan tembok , ngadep ke lift \" \" ha ? Cewe ?',\n",
       " 'Ga ada siapa2 ah ca \" bantah nay tiba2 . \" ih , eca liat sendiri nay . cewenya pake',\n",
       " 'kemeja putih , rambut di kuncir , pake celana , ngeliat ke kita . Makanya eca',\n",
       " 'bilang untung ada orang \" \" ga ada caaaaa ga ada orang \" bantah kami berdua .',\n",
       " 'Tiba2 kakak senior di sebelah kami yang tidak sengaja mendengar percakapan',\n",
       " 'kami tadi angkat bicara , \" hmm lantai 7 ? Pake lift sebelah kiri ya ?\" \" iya \" kata',\n",
       " 'ku . Dia hanya mengangguk , \" hmm pantesan , lain kali hati2 ya \".',\n",
       " '']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outer[-100:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dumping-watpadd.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(outer))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
