{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fake_useragent import UserAgent\n",
    "import requests\n",
    "\n",
    "ua = UserAgent()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from newspaper import Article\n",
    "from queue import Queue\n",
    "from urllib.parse import quote\n",
    "from unidecode import unidecode\n",
    "\n",
    "def get_date(load):\n",
    "    try:\n",
    "        date = re.findall(\n",
    "            '[-+]?[.]?[\\d]+(?:,\\d\\d\\d)*[\\.]?\\d*(?:[eE][-+]?\\d+)?', load\n",
    "        )\n",
    "        return '%s-%s-%s' % (date[2], date[0], date[1])\n",
    "    except Exce:\n",
    "        return False\n",
    "\n",
    "\n",
    "def run_parallel_in_threads(target, args_list):\n",
    "    globalparas = []\n",
    "    result = Queue()\n",
    "\n",
    "    def task_wrapper(*args):\n",
    "        result.put(target(*args))\n",
    "\n",
    "    threads = [\n",
    "        threading.Thread(target = task_wrapper, args = args)\n",
    "        for args in args_list\n",
    "    ]\n",
    "    for t in threads:\n",
    "        t.start()\n",
    "    for t in threads:\n",
    "        t.join()\n",
    "    while not result.empty():\n",
    "        globalparas.append(result.get())\n",
    "    globalparas = list(filter(None, globalparas))\n",
    "    return globalparas\n",
    "\n",
    "\n",
    "def get_article(link, news, date):\n",
    "    article = Article(link)\n",
    "    article.download()\n",
    "    article.parse()\n",
    "    article.nlp()\n",
    "    lang = 'ENGLISH'\n",
    "    if len(article.title) < 5 or len(article.text) < 5:\n",
    "        lang = 'INDONESIA'\n",
    "        print('found BM/ID article')\n",
    "        article = Article(link, language = 'id')\n",
    "        article.download()\n",
    "        article.parse()\n",
    "        article.nlp()\n",
    "    return {\n",
    "        'title': article.title,\n",
    "        'url': link,\n",
    "        'authors': article.authors,\n",
    "        'top-image': article.top_image,\n",
    "        'text': article.text,\n",
    "        'keyword': article.keywords,\n",
    "        'summary': article.summary,\n",
    "        'news': news,\n",
    "        'date': date,\n",
    "        'language': lang,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT = 0\n",
    "\n",
    "GOOGLE_NEWS_URL = 'https://www.google.com.my/search?q={}&source=lnt&tbs=cdr%3A1%2Ccd_min%3A{}%2Ccd_max%3A{}&tbm=nws&start={}'\n",
    "\n",
    "def forge_url(q, start, year_start, year_end):\n",
    "    global NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT\n",
    "    NUMBER_OF_CALLS_TO_GOOGLE_NEWS_ENDPOINT += 1\n",
    "    return GOOGLE_NEWS_URL.format(\n",
    "        q.replace(' ', '+'), str(year_start), str(year_end), start\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.google.com.my/search?q=isu+syarikat&source=lnt&tbs=cdr%3A1%2Ccd_min%3A2005%2Ccd_max%3A2021&tbm=nws&start=0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_articles_index = 0\n",
    "url = forge_url('isu syarikat', num_articles_index, '2005', '2021')\n",
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}\n",
    "headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(url, headers = headers, timeout = 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime, timedelta\n",
    "from dateutil import parser\n",
    "\n",
    "def extract_links(content):\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "    # return soup\n",
    "    today = datetime.now().strftime('%m/%d/%Y')\n",
    "    links_list = [\n",
    "        v.attrs['href'] for v in soup.find_all('a', {'class': 'l lLrAF'.split()})\n",
    "    ]\n",
    "    dates_list = [v.text for v in soup.find_all('div', {'class': 'dhIWPd'.split()})]\n",
    "    output = []\n",
    "    for (link, date) in zip(links_list, dates_list):\n",
    "        try:\n",
    "            date = date.split('-')\n",
    "            if (\n",
    "                date[1].find('hour') >= 0\n",
    "                or date[1].find('minute') >= 0\n",
    "                or date[1].find('分鐘') >= 0\n",
    "                or date[1].find('小時') >= 0\n",
    "            ):\n",
    "                date[1] = today\n",
    "            elif date[1].find('day') >= 0 or date[1].find('日') >= 0:\n",
    "                count = date[1].split(' ')[0]\n",
    "            else:\n",
    "                try:\n",
    "                    date[1] = parser.parse(date[1]).strftime('%m-%d-%Y')\n",
    "                except:\n",
    "                    date[1] = 'null'\n",
    "            output.append((link, date[0].strip(), date[1]))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# response.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('test.html', 'w') as fopen:\n",
    "#     fopen.write(response.content.decode())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('https://www.bharian.com.my/bisnes/lain-lain/2019/07/582523/isu-malaysia-airlines-perlu-diselesai-segera',\n",
       "  'Berita Harian',\n",
       "  '07-08-2019'),\n",
       " ('https://www.bharian.com.my/berita/nasional/2019/03/540041/kerajaan-pertimbang-tutup-malaysia-airlines',\n",
       "  'Berita Harian',\n",
       "  '03-11-2019'),\n",
       " ('https://www.malaysiakini.com/letters/506320',\n",
       "  'Malaysiakini (caruman)',\n",
       "  '01-07-2020'),\n",
       " ('https://www.malaysiakini.com/news/491861',\n",
       "  'Malaysiakini (caruman)',\n",
       "  '09-13-2019'),\n",
       " ('https://www.bharian.com.my/bisnes/lain-lain/2019/10/623152/pnb-serius-tangani-isu-tadbir-urus-syarikat-pelaburan',\n",
       "  'Berita Harian',\n",
       "  '10-30-2019'),\n",
       " ('https://www.bharian.com.my/berita/nasional/2019/11/627385/isu-keselamatan-penerbangan-caam-akui-wujud-kelemahan',\n",
       "  'Berita Harian',\n",
       "  '11-11-2019'),\n",
       " ('https://www.hmetro.com.my/mutakhir/2019/10/502972/tiga-kementerian-kaji-isu-foodpanda',\n",
       "  'Harian Metro',\n",
       "  '10-02-2019'),\n",
       " ('https://www.bharian.com.my/berita/nasional/2019/10/613557/grab-didenda-lebih-rm86-juta',\n",
       "  'Berita Harian',\n",
       "  '10-02-2019'),\n",
       " ('https://www.bharian.com.my/hiburan/selebriti/2019/10/619353/biar-syarikat-rakaman-ambil-tindakan-sirkhan',\n",
       "  'Berita Harian',\n",
       "  '10-18-2019'),\n",
       " ('https://www.bharian.com.my/berita/nasional/2020/04/676678/pkp-isu-dan-jawapan-berkaitan-industri-dibenarkan-miti',\n",
       "  'Berita Harian',\n",
       "  '04-13-2020')]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = extract_links(response.content)\n",
    "r"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
