{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pyscisci.all as pyscisci\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set this path to where the DBLP database will be stored\n",
    "path2dblp = '/home/ajgates/DBLP'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mydblp = pyscisci.DBLP(path2dblp, database_extension='csv.gz', keep_in_memory=False) \n",
    "\n",
    "# set keep_in_memory=False if you want to load the database each time its needed - good for when you \n",
    "# cant keep more than one DataFrame in memory at a time\n",
    "\n",
    "# otherwise keep_in_memory=True will keep each DataFrame in memory after its loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f912741ad204293a168cb7c6924e315",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='dblp.xml.gz', max=588476239.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# only need to run this for the first time\n",
    "# usually takes about 1-2min to download\n",
    "mydblp.download_from_source()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting to preprocess the DBLP database.\n",
      "Xml tree parsed, iterating through elements.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99e5b91d53084f468384a28056a2375a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='dblp.xml', max=3003721560.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ajgates/.pyenv/versions/3.7.3/lib/python3.7/site-packages/pandas/core/generic.py:2505: PerformanceWarning: \n",
      "your performance may suffer as PyTables will pickle object types that it cannot\n",
      "map directly to c-types [inferred_type->mixed,key->block2_values] [items->Index(['Title', 'Number', 'Pages', 'JournalId', 'EE', 'DocType', 'URL'], dtype='object')]\n",
      "\n",
      "  encoding=encoding,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# before we can start running our analysis, we have to preprocess the raw data into\n",
    "# DataFrames that are more convinent to work with\n",
    "mydblp.preprocess()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DBLP creates three dataframes:\n",
    "# pub  - keeps all of the publication information\n",
    "# columns : ['PublicationId', 'Title', 'Year', 'Month', 'Volume', 'Number', 'Pages', 'JournalId', 'EE', 'DocType', TeamSize']\n",
    "\n",
    "# author  - keeps all of the author information\n",
    "# columns : ['AuthorId', 'FullName', 'LastName', 'FirstName', 'MiddleName']\n",
    "\n",
    "# author2pub  - links the authors to their publications\n",
    "# columns : ['PublicationId', 'AuthorId', 'AuthorSequence']\n",
    "\n",
    "# NOTE: DBLP does not contain citation information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c457645dc4b4f38bb9a233a2f8a3da2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Loading Publications', max=6.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/site-packages/tqdm/std.py:668: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n",
      "/usr/local/Cellar/graph-tool/2.33/libexec/lib/python3.8/site-packages/matplotlib/cbook/__init__.py:1402: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version.  Convert to a numpy array before indexing instead.\n",
      "  x[:, None]\n",
      "/usr/local/Cellar/graph-tool/2.33/libexec/lib/python3.8/site-packages/matplotlib/axes/_base.py:276: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version.  Convert to a numpy array before indexing instead.\n",
      "  x = x[:, np.newaxis]\n",
      "/usr/local/Cellar/graph-tool/2.33/libexec/lib/python3.8/site-packages/matplotlib/axes/_base.py:278: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version.  Convert to a numpy array before indexing instead.\n",
      "  y = y[:, np.newaxis]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19044 characters]\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# lets plot the number of publications each year\n",
    "yearly_articles = pyscisci.groupby_count(df=mydblp.pub , \n",
    "                                colgroupby='Year', \n",
    "                                colcountby='PublicationId', \n",
    "                                count_unique=True)\n",
    "yearly_articles.sort_values(by='Year', inplace=True)\n",
    "yearly_articles = yearly_articles.loc[yearly_articles['Year'] > 0]\n",
    "\n",
    "fig, ax = plt.subplots(1,1,figsize=(8,5))\n",
    "\n",
    "ax.plot(yearly_articles['Year'], yearly_articles['PublicationIdCount'])\n",
    "\n",
    "ax.set_xlabel('Year')\n",
    "ax.set_ylabel(\"# of publications\")\n",
    "ax.set_yscale('log')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b7077e58fd6497d82e2f34e3b5c6d2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Loading PublicationAuthor', max=6.0, style=ProgressStyle(…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/site-packages/tqdm/std.py:668: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12128 characters]\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# now we can see the distribution of author productivity \n",
    "\n",
    "pa  = mydblp.author2pub  # publication author relationships\n",
    "\n",
    "author_prod = pyscisci.author_productivity(pa , colgroupby = 'AuthorId', colcountby = 'PublicationId')\n",
    "\n",
    "prodvalues, prodcounts = np.unique(author_prod['Productivity'].values, return_counts=True)\n",
    "\n",
    "fig, ax = plt.subplots(1,1,figsize=(8,5))\n",
    "\n",
    "ax.scatter(prodvalues, prodcounts)\n",
    "\n",
    "ax.set_xlabel('Productivity')\n",
    "ax.set_ylabel(\"# of authors\")\n",
    "ax.set_xscale('log')\n",
    "ax.set_yscale('log')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
