{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 1: Productivity of a Scientist\n",
    "\n",
    "Note: Here we use the MAG, a much larger database than used for most figures in the textbook.  We expect for specific numbers to differ from the exact figures shown in the textbook, but all trends and distributions to remain unchanged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# headers\n",
    "import pyscisci.all as pyscisci\n",
    "\n",
    "import numpy as np\n",
    "import scipy.stats as spstats\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some useful functions and definitions\n",
    "red_color = '#f27c96'\n",
    "lightblue_color = '#7cd0ea'\n",
    "darkblue_color = '#154959'\n",
    "green_color = '#93d0aa'\n",
    "\n",
    "# a helper function to fit the lognormal distribution using Scipy's maximum likelihood fitter\n",
    "def fit_lognorm(data):\n",
    "    # get initial guess:\n",
    "    mu_guess = np.log( np.mean(data)**2 / np.sqrt(np.var(data) + np.mean(data)**2) )\n",
    "    sigma_guess = np.log( 1 + np.var(data) / np.mean(data)**2 )\n",
    "    \n",
    "    # fit will the numberical initial guess for higher accuracy\n",
    "    lnrv_s, lnrv_loc, lnrv_scale = spstats.lognorm.fit(data, \n",
    "                                                       loc=mu_guess, \n",
    "                                                       s=sigma_guess, scale=1)\n",
    "    \n",
    "    # get the log-likelihood for the distribution fit\n",
    "    lnrv_llh = spstats.lognorm.logpdf(data, lnrv_s, lnrv_loc, lnrv_scale).sum()\n",
    "    return (lnrv_s, lnrv_loc, lnrv_scale, lnrv_llh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you have two options for the dataset to use\n",
    "\n",
    "# Option 1: MAG - larger and comprehensive\n",
    "if False:\n",
    "    # make sure you have first run Chapter 0 (only need to do once)\n",
    "    # then set this path to the location where the processed MAG data are stored\n",
    "    path2mag = '/home/ajgates/MAG'\n",
    "    path2mag = \"/Volumes/GatesExpansionDrive/DataSets/MAG\"\n",
    "\n",
    "    # to accurately reproduce the calculations in the textbook, we must limit our data to only journal articles \n",
    "    # denoted in pySciSci by DocType == 'j'\n",
    "\n",
    "    journal_paper_filter = pyscisci.DocTypeFilter(doctypes = ['j'])\n",
    "\n",
    "    # now create the database object\n",
    "    mydata = pyscisci.MAG(path2mag, global_filter=journal_paper_filter)\n",
    "\n",
    "# Option 2: DBLP - smaller and much faster    \n",
    "else:\n",
    "    \n",
    "    # make sure you have first run Chapter 0 (only need to do once)\n",
    "    # then set this path to the location where the processed DBLP data are stored\n",
    "    path2dblp = '/home/ajgates/DBLP'\n",
    "    path2dblp = \"/Volumes/GatesExpansionDrive/DataSets/DBLP\"\n",
    "\n",
    "    # now create the database object\n",
    "    mydata = pyscisci.DBLP(path2dblp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2141c456d7764f44b16968cac56f2dcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Loading Publication Author Affiliation'), FloatProgress(value=0.0, max=63.0), HTML(…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AuthorId</th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2582736345</td>\n",
       "      <td>23</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2936336055</td>\n",
       "      <td>23</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2938518172</td>\n",
       "      <td>23</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2662843304</td>\n",
       "      <td>79</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2683440697</td>\n",
       "      <td>79</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     AuthorId  PublicationId  Year\n",
       "3  2582736345             23  2012\n",
       "4  2936336055             23  2012\n",
       "5  2938518172             23  2012\n",
       "8  2662843304             79  2009\n",
       "9  2683440697             79  2009"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# now load the DataFrame containing just authors and their publications\n",
    "author_publications = mydata.author2pub_df\n",
    "\n",
    "# we will also need the publication Year, which is stored in both the publication DataFrame and as a dictionary\n",
    "pub2year = mydata.pub2year\n",
    "\n",
    "# create a new column with the publication year information\n",
    "author_publications['Year'] = [pub2year.get(pid, None) for pid in author_publications['PublicationId'].values]\n",
    "\n",
    "# and print out the top of the DataFrame so you can see what columns look like\n",
    "author_publications.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 1.1 The growing number of scientists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ajgates/.pyenv/versions/3.9.0/lib/python3.9/site-packages/tqdm/std.py:699: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf343262dea442b8b837d5cca9270686",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=221.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9e2d5b20a6b4395943325657fc360b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=221.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 1.1a - publications\n",
    "# count the number of publications each year \n",
    "# (calculated by a groupby on years and count the unique publication ids)\n",
    "number_yearly_publications = pyscisci.groupby_count(author_publications, colgroupby='Year', colcountby='PublicationId', \n",
    "                       count_unique=True, show_progress=True)\n",
    "\n",
    "number_yearly_publications.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b879eaf09b4009850b5ab355b1e483",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=221.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>AuthorIdCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012</td>\n",
       "      <td>5867776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2009</td>\n",
       "      <td>4920701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013</td>\n",
       "      <td>6232303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2008</td>\n",
       "      <td>4528013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1986</td>\n",
       "      <td>1424561</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  AuthorIdCount\n",
       "0  2012        5867776\n",
       "1  2009        4920701\n",
       "2  2013        6232303\n",
       "3  2008        4528013\n",
       "4  1986        1424561"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.1a - authors\n",
    "# count the number of authors each year \n",
    "# (calculated by a groupby on years and count the unique author ids)\n",
    "number_yearly_authors = pyscisci.groupby_count(author_publications, colgroupby='Year', colcountby='AuthorId', \n",
    "                       count_unique=True, show_progress=True)\n",
    "\n",
    "number_yearly_authors.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ajgates/.pyenv/versions/3.9.0/lib/python3.9/site-packages/tqdm/std.py:699: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5808eaaf735749a1a7ea4033823bb8cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=84091109.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "module 'pyscisci.all' has no attribute 'groupby_mean'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-d021d6861fff>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;31m# now average\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m number_yearly_authors_a_publication = pyscisci.groupby_mean(authors_per_publication, colgroupby='Year', \n\u001b[0m\u001b[1;32m     15\u001b[0m                                                             colcountby='AuthorIdCount', show_progress=True)\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'pyscisci.all' has no attribute 'groupby_mean'"
     ]
    }
   ],
   "source": [
    "# 1.1b - authors per publications\n",
    "# first count the number of authors on each publication\n",
    "# (calculated by a groupby on years and count the unique publication ids)\n",
    "\n",
    "# this is also pre-computed as 'TeamSize' in the publication data frame, but for completeness\n",
    "# we will calculate it again here\n",
    "authors_per_publication = pyscisci.groupby_count(author_publications, colgroupby='PublicationId', \n",
    "                                                 colcountby='AuthorId', \n",
    "                                                   count_unique=True, show_progress=True)\n",
    "# bring back the year info\n",
    "authors_per_publication['Year'] = [pub2year.get(pid, None) for pid in authors_per_publication['PublicationId'].values]\n",
    "\n",
    "# now average\n",
    "number_yearly_authors_a_publication = pyscisci.groupby_mean(authors_per_publication, colgroupby='Year', \n",
    "                                                            colcountby='AuthorIdCount', show_progress=True)\n",
    "\n",
    "number_yearly_authors_a_publication.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.1b - publications per author\n",
    "# first count the number of publications per author per year\n",
    "# (calculated by a groupby on years and count the unique publication ids)\n",
    "\n",
    "# this is known as the yearly productivity\n",
    "publications_per_author_per_year = pyscisci.author_yearly_productivity(author_publications, \n",
    "                                                              colgroupby = 'AuthorId', \n",
    "                                                              datecol = 'Year', \n",
    "                                                              colcountby = 'PublicationId', \n",
    "                                                              show_progress = True)\n",
    "\n",
    "# now average\n",
    "number_yearly_publications_per_author = pyscisci.groupby_mean(publications_per_author_per_year, colgroupby='Year', \n",
    "                                                            colcountby='PublicationIdCount', show_progress=True)\n",
    "\n",
    "number_yearly_publications_per_author.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,2, figsize=(12, 5))\n",
    "\n",
    "ax[0].set_title('a) Overall')\n",
    "\n",
    "ax[0].plot(number_yearly_publications['Year'].values, \n",
    "           number_yearly_publications['PublicationIdCount'].values,\n",
    "          color=red_color, ls='-', label='#papers')\n",
    "\n",
    "ax[0].plot(number_yearly_authors['Year'].values, \n",
    "           number_yearly_authors['AuthorIdCount'].values,\n",
    "          color=lightblue_color, ls='--', label='#authors')\n",
    "\n",
    "ax[0].set_xlim([1900, 2020])\n",
    "ax[0].set_xlabel('Year')\n",
    "ax[0].set_ylabel('Overall number')\n",
    "ax[0].set_yscale('log')\n",
    "\n",
    "\n",
    "ax[1].set_title('b) Average')\n",
    "\n",
    "ax[1].plot(number_yearly_authors_a_publication['Year'].values, \n",
    "           number_yearly_authors_a_publication['PublicationIdCount'].values,\n",
    "          color=red_color, ls='-', label='#authors/papers')\n",
    "\n",
    "ax[1].plot(number_yearly_publications_per_author['Year'].values, \n",
    "           number_yearly_publications_per_author['AuthorIdCount'].values,\n",
    "          color=lightblue_color, ls='--', label='#papers/authors')\n",
    "\n",
    "ax[1].set_xlim([1900, 2020])\n",
    "ax[1].set_xlabel('Year')\n",
    "ax[1].set_ylim([0,5])\n",
    "ax[1].set_ylabel('Average number')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure 1.2 Productivity Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.2a - raw distribution\n",
    "# count the number of publications per author\n",
    "# this is known as the productivity\n",
    "number_publications_per_author = pyscisci.author_productivity(author_publications, \n",
    "                                                              colgroupby = 'AuthorId', \n",
    "                                                              colcountby = 'PublicationId', \n",
    "                                                              show_progress=True)\n",
    "\n",
    "\n",
    "# then we need the number of authors that have the same number of publications\n",
    "productivity_values, author_counts = np.unique(number_publications_per_author['Productivity'].values, \n",
    "                                               return_counts=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# next we can try fitting the lognormal distribution to the raw productivity data \n",
    "# (see cell 2 for def of fit_lognorm)\n",
    "lnrv_s, lnrv_loc, lnrv_scale, lnrv_llh = fit_lognorm(number_publications_per_author['Productivity'].values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,1, figsize=(6, 5))\n",
    "\n",
    "\n",
    "ax.scatter(productivity_values, \n",
    "           author_counts,\n",
    "          color=lightblue_color, marker='s', label='all authors')\n",
    "\n",
    "\n",
    "lognorm_xs = productivity_values\n",
    "lognorm_ys = author_counts.sum()*spstats.lognorm(lnrv_s, lnrv_loc, lnrv_scale).pdf(lognorm_xs)\n",
    "\n",
    "ax.plot(lognorm_xs, lognorm_ys,\n",
    "          color=red_color, ls='-', label='log-norm. distr.')\n",
    "\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlabel('Number of publications x')\n",
    "ax.set_ylabel('N(x) number of authors with publications')\n",
    "ax.set_yscale('log')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
