{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import dependencies\n",
    "from elasticsearch import Elasticsearch\n",
    "import wikipediaapi\n",
    "from slugify import slugify\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = Elasticsearch(\"localhost:9200\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pandemics': {'aliases': {}},\n",
       " 'american-science-fiction-television-series': {'aliases': {}},\n",
       " 'coronaviridae': {'aliases': {}},\n",
       " 'natural-language-processing': {'aliases': {}},\n",
       " 'american-comics-writers': {'aliases': {}},\n",
       " 'marvel-comics': {'aliases': {}},\n",
       " 'marvel-comics-editors-in-chief': {'aliases': {}},\n",
       " 'presidents-of-the-united-states': {'aliases': {}},\n",
       " 'machine-learning': {'aliases': {}},\n",
       " '21st-century-american-comedians': {'aliases': {}}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.indices.get_alias(\"_all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.indices.delete('_all')\n",
    "client.indices.get_alias(\"_all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {\n",
    "    \"properties\": {\n",
    "        \n",
    "            \"text\": {\n",
    "                \"type\": \"nested\",\n",
    "                \"properties\":{\n",
    "                    \"section_num\": {\"type\":\"integer\"},\n",
    "                    \"section_title\": {\"type\":\"text\"},\n",
    "                    \"section_content\": {\"type\":\"text\"}\n",
    "                }\n",
    "            },\n",
    "        \n",
    "            \"title\": {\n",
    "                \"type\": \"text\"\n",
    "            },\n",
    "        \n",
    "            \"source\": {\n",
    "                \"type\": \"text\"\n",
    "            },\n",
    "        \n",
    "            \"page_id\": {\n",
    "                \"type\": \"long\"\n",
    "            },\n",
    "            \n",
    "        }\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['Presidents of the United States', \n",
    "              'Marvel Comics', \n",
    "              'American comics writers',\n",
    "              'Marvel Comics editors-in-chief']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load wikipedia articles from the following categories:\n",
    "categories2 = ['Machine learning',\n",
    "              'Natural language processing',\n",
    "              'Coronaviridae',\n",
    "              '21st-century American comedians',\n",
    "              'Pandemics']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Document:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.title = ''\n",
    "        self.page_id = None\n",
    "        self.source = ''\n",
    "        self.text = ''\n",
    "        \n",
    "    def __if_exists(self, page_id, index=\"\"):\n",
    "        '''\n",
    "        A private method to check if the article already exists in the database\n",
    "        with a goal to avoid duplication\n",
    "        '''\n",
    "        \n",
    "        return client.search(index=index, \n",
    "                             body={\"query\": \n",
    "                                   {\"match\": \n",
    "                                    {\"page_id\": page_id}\n",
    "                                   }})['hits']['total']['value']\n",
    "        \n",
    "    def insert(self, title, page_id, url, text, index):\n",
    "        ''' Add a new document to the index'''\n",
    "        \n",
    "        self.title=title\n",
    "        self.page_id=page_id\n",
    "        self.source=url\n",
    "        self.text=text\n",
    "        self.body = {'title': self.title,\n",
    "            'page_id': self.page_id,\n",
    "            'source':self.source,\n",
    "            'text': self.text}\n",
    "        \n",
    "        if self.__if_exists(page_id) == 0:\n",
    "        \n",
    "            try:\n",
    "                client.index(index=index, body=self.body)\n",
    "#                 print(f\"Sucess! The article {self.title} was added to index {index}\")\n",
    "            except error:\n",
    "                print(\"Something went wrong\", error)\n",
    "                \n",
    "        else:\n",
    "            print(f\"Article {self.title} is already in the database\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_article(article):\n",
    "    ''' Parce wikipedia articles from the full article text'''\n",
    "    \n",
    "    text = article.text\n",
    "    # get section titles for the existing sections\n",
    "    section_titles = [sec.title for sec in article.sections]\n",
    "    \n",
    "    # initiate the sections dictionary with a summary (0th section) \n",
    "    sections = [{'section_num': 0},\n",
    "                {'section_title': \"Summary\"},\n",
    "                {'section_content': article.summary}]\n",
    "    \n",
    "    for i, title in enumerate(section_titles[::-1]):\n",
    "\n",
    "        num = len(section_titles)-i\n",
    "        if len(text.split(f\"\\n\\n{title}\")) == 2:\n",
    "            section_dict = {\"section_num\": num,\n",
    "                            \"section_title\": title,\n",
    "                            \"section_content\": text.split(f\"\\n\\n{title}\")[-1]}\n",
    "            sections.append(section_dict)\n",
    "            text = text.split(f\"\\n\\n{title}\")[0]\n",
    "        else:\n",
    "            pass\n",
    "            \n",
    "        \n",
    "    return sections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_insert_wiki(category, mapping):\n",
    "    \n",
    "    if type(category) is not list: category = [ category ]\n",
    "\n",
    "    wiki_wiki = wikipediaapi.Wikipedia('en')\n",
    "    \n",
    "    for c in category:\n",
    "        \n",
    "        try:\n",
    "                    \n",
    "            '''Create and empty index with predefined data structure'''\n",
    "            client.indices.create(index=slugify(c), body={\"mappings\":mapping})\n",
    "            \n",
    "            '''Access the list of wikipedia articles in category c'''\n",
    "            cat = wiki_wiki.page(f\"Category:{c}\")\n",
    "            \n",
    "            ''' Parse and add articles in the category to database'''\n",
    "            for key in cat.categorymembers.keys():\n",
    "                page = wiki_wiki.page(key)\n",
    "\n",
    "                if not \"Category:\" in page.title:\n",
    "\n",
    "                    text = parse_article(page)\n",
    "                    doc = Document()\n",
    "                    doc.insert(page.title, page.pageid, page.fullurl, text, index=slugify(c))\n",
    "\n",
    "\n",
    "        except error:\n",
    "            '''Skip category if it alredy exists in indices'''\n",
    "            print(\"Something went wrong\", error)\n",
    "            \n",
    "search_insert_wiki(categories, mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_wiki_doc('Coronaviridae')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "client.indices.get_alias(\"_all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "client.cat.count('coronaviridae', params={\"format\": \"json\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "body = {\n",
    "    'size': 20,\n",
    "    'query': {\n",
    "        'match_all': 'Stephen Colbert'\n",
    "    }}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "RequestError",
     "evalue": "RequestError(400, 'parsing_exception', '[match_all] query malformed, no start_object after query name')",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRequestError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-e0f8911b025c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbody\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Anaconda3\\envs\\notebooks\\lib\\site-packages\\elasticsearch\\client\\utils.py\u001b[0m in \u001b[0;36m_wrapped\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     82\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mp\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m                     \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0m_wrapped\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\notebooks\\lib\\site-packages\\elasticsearch\\client\\__init__.py\u001b[0m in \u001b[0;36msearch\u001b[1;34m(self, body, index, doc_type, params)\u001b[0m\n\u001b[0;32m   1545\u001b[0m             \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"from\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"from_\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1546\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1547\u001b[1;33m         return self.transport.perform_request(\n\u001b[0m\u001b[0;32m   1548\u001b[0m             \u001b[1;34m\"GET\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_make_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdoc_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"_search\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1549\u001b[0m         )\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\notebooks\\lib\\site-packages\\elasticsearch\\transport.py\u001b[0m in \u001b[0;36mperform_request\u001b[1;34m(self, method, url, headers, params, body)\u001b[0m\n\u001b[0;32m    349\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    350\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 351\u001b[1;33m                 status, headers_response, data = connection.perform_request(\n\u001b[0m\u001b[0;32m    352\u001b[0m                     \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    353\u001b[0m                     \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\notebooks\\lib\\site-packages\\elasticsearch\\connection\\http_urllib3.py\u001b[0m in \u001b[0;36mperform_request\u001b[1;34m(self, method, url, params, body, timeout, ignore, headers)\u001b[0m\n\u001b[0;32m    259\u001b[0m                 \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfull_url\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mduration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mraw_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    260\u001b[0m             )\n\u001b[1;32m--> 261\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_raise_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mraw_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    262\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    263\u001b[0m         self.log_request_success(\n",
      "\u001b[1;32mC:\\Anaconda3\\envs\\notebooks\\lib\\site-packages\\elasticsearch\\connection\\base.py\u001b[0m in \u001b[0;36m_raise_error\u001b[1;34m(self, status_code, raw_data)\u001b[0m\n\u001b[0;32m    179\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Undecodable raw error response from server: %s\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    180\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 181\u001b[1;33m         raise HTTP_EXCEPTIONS.get(status_code, TransportError)(\n\u001b[0m\u001b[0;32m    182\u001b[0m             \u001b[0mstatus_code\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merror_message\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0madditional_info\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    183\u001b[0m         )\n",
      "\u001b[1;31mRequestError\u001b[0m: RequestError(400, 'parsing_exception', '[match_all] query malformed, no start_object after query name')"
     ]
    }
   ],
   "source": [
    "res = client.search(index = '', body = body )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, r in enumerate(res['hits']['hits'], start = 1):\n",
    "    print(i, '--', r['_source']['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nbooks] *",
   "language": "python",
   "name": "conda-env-nbooks-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
