{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "import os\n",
    "import json\n",
    "import itertools\n",
    "\n",
    "def clearstring(string):\n",
    "    string = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '',' '.join([i for i in string.split() if i.find('#')<0 and i.find('@')<0]))\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('[^A-Za-z\\- ]+', '', string)\n",
    "    string = word_tokenize(string)\n",
    "    string = filter(None, string)\n",
    "    string = [y.strip() for y in string if len(y)> 1]\n",
    "    string = ' '.join(string).lower()\n",
    "    return ''.join(''.join(s)[:2] for _, s in itertools.groupby(string))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news = [i for i in os.listdir(os.getcwd()) if i.find('isu') >= 0]\n",
    "len(news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "isu barisan nasional.json 2164\n",
      "isu air.json 1628\n",
      "isu astro.json 2336\n",
      "isu sosial media.json 2773\n",
      "isu sultan melayu.json 1221\n",
      "isu tm.json 2261\n",
      "isu pelakon.json 414\n",
      "isu teknologi.json 3683\n",
      "isu sekolah.json 2239\n",
      "isu kerajaan.json 2342\n",
      "isu mahathir.json 2000\n",
      "isu artis.json 1676\n",
      "isu politik.json 3090\n",
      "isu najib razak.json 1306\n",
      "isu.json 3068\n",
      "isu minyak.json 2086\n",
      "isu sosial.json 3403\n",
      "isu pembangkang.json 2742\n",
      "isu lgbt.json 1689\n",
      "isu agama.json 945\n",
      "isu bahasa melayu.json 1636\n",
      "isu ekonomi.json 1080\n",
      "isu dunia.json 2112\n",
      "isu malaysia.json 2606\n",
      "isu rosmah.json 1665\n",
      "isu harga.json 2656\n",
      "isu kesihatan.json 2736\n"
     ]
    }
   ],
   "source": [
    "combined = []\n",
    "for n in news:\n",
    "    with open(n,'r') as fopen:\n",
    "        data = json.loads(fopen.read())\n",
    "    data = [i for i in data if i['language'] != 'eng']\n",
    "    texts=[i['text'] for i in data]\n",
    "    texts=' '.join(texts)\n",
    "    texts=re.sub(r'(?<!\\d)\\.(?!\\d)', 'SPLITTT', texts.replace('\\n','').replace('/',' '))\n",
    "    texts=texts.split('SPLITTT')\n",
    "    print(n,len(texts))\n",
    "    combined += [clearstring(i) for i in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "57557"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('news-bm.json','w') as fopen:\n",
    "    fopen.write(json.dumps(combined))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
