{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chapter 2: The h-index\n",
    "\n",
    "Note: Here we use the APS dataset with the author disambiguation used in Sinatra et al (2016).  We expect for specific numbers to differ from the exact figures shown in the textbook, but all trends and distributions to remain unchanged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# headers\n",
    "import pyscisci.all as pyscisci\n",
    "\n",
    "import numpy as np\n",
    "import scipy.stats as spstats\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "\n",
    "# some useful functions and definitions\n",
    "red_color = '#f27c96'\n",
    "lightblue_color = '#7cd0ea'\n",
    "darkblue_color = '#154959'\n",
    "green_color = '#93d0aa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pyscisci.filter.DocTypeFilter object at 0x1234e7730>\n",
      "{'DocType': <pyscisci.filter.DocTypeFilter object at 0x1234e7730>}\n",
      "call load\n",
      "<class 'dict'>\n"
     ]
    }
   ],
   "source": [
    "# make sure you have first run Chapter 0 (only need to do once)\n",
    "# then set this path to the location where the processed MAG data are stored\n",
    "path2aps = '/home/ajgates/APS'\n",
    "path2mag = \"/Volumes/GatesExpansionDrive/DataSets/MAG\"\n",
    "\n",
    "path2aps = '/Volumes/GatesExpansionDrive/DataSets/APS/APS2019'\n",
    "\n",
    "# now create the database object\n",
    "myaps = pyscisci.APS(path2aps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2141c456d7764f44b16968cac56f2dcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Loading Publication Author Affiliation'), FloatProgress(value=0.0, max=63.0), HTML(…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AuthorId</th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2582736345</td>\n",
       "      <td>23</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2936336055</td>\n",
       "      <td>23</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2938518172</td>\n",
       "      <td>23</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2662843304</td>\n",
       "      <td>79</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2683440697</td>\n",
       "      <td>79</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     AuthorId  PublicationId  Year\n",
       "3  2582736345             23  2012\n",
       "4  2936336055             23  2012\n",
       "5  2938518172             23  2012\n",
       "8  2662843304             79  2009\n",
       "9  2683440697             79  2009"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# now load the DataFrame containing just authors and their publications\n",
    "author_publications = myaps.author2pub_df\n",
    "\n",
    "# we will also need the publication Year, which is stored in both the publication DataFrame and as a dictionary\n",
    "pub2year = mymag.pub2year\n",
    "\n",
    "# create a new column with the publication year information\n",
    "author_publications['Year'] = [pub2year.get(pid, None) for pid in author_publications['PublicationId'].values]\n",
    "\n",
    "# and print out the top of the DataFrame so you can see what columns look like\n",
    "author_publications.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 2.1 The h-index of Albert Einstein and Peter Higgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the AuthorIds for the two focus authors\n",
    "Albert_Einstein_AuthorId = 1782324548\n",
    "Peter_Higgs_AuthorId = 2584434824"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ajgates/.pyenv/versions/3.9.0/lib/python3.9/site-packages/tqdm/std.py:699: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf343262dea442b8b837d5cca9270686",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=221.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9e2d5b20a6b4395943325657fc360b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=221.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 1.1a - publications\n",
    "# count the number of publications each year \n",
    "# (calculated by a groupby on years and count the unique publication ids)\n",
    "number_yearly_publications = pyscisci.groupby_count(author_publications, colgroupby='Year', colcountby='PublicationId', \n",
    "                       count_unique=True, show_progress=True)\n",
    "\n",
    "number_yearly_publications.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b879eaf09b4009850b5ab355b1e483",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=221.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>AuthorIdCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012</td>\n",
       "      <td>5867776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2009</td>\n",
       "      <td>4920701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013</td>\n",
       "      <td>6232303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2008</td>\n",
       "      <td>4528013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1986</td>\n",
       "      <td>1424561</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  AuthorIdCount\n",
       "0  2012        5867776\n",
       "1  2009        4920701\n",
       "2  2013        6232303\n",
       "3  2008        4528013\n",
       "4  1986        1424561"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.1a - authors\n",
    "# count the number of authors each year \n",
    "# (calculated by a groupby on years and count the unique author ids)\n",
    "number_yearly_authors = pyscisci.groupby_count(author_publications, colgroupby='Year', colcountby='AuthorId', \n",
    "                       count_unique=True, show_progress=True)\n",
    "\n",
    "number_yearly_authors.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pyscisci' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-d021d6861fff>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# this is also pre-computed as 'TeamSize' in the publication data frame, but for completeness\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# we will calculate it again here\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m authors_per_publication = pyscisci.groupby_count(author_publications, colgroupby='PublicationId', \n\u001b[0m\u001b[1;32m      8\u001b[0m                                                  \u001b[0mcolcountby\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'AuthorId'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m                                                    count_unique=True, show_progress=True)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pyscisci' is not defined"
     ]
    }
   ],
   "source": [
    "# 1.1b - authors per publications\n",
    "# first count the number of authors on each publication\n",
    "# (calculated by a groupby on years and count the unique publication ids)\n",
    "\n",
    "# this is also pre-computed as 'TeamSize' in the publication data frame, but for completeness\n",
    "# we will calculate it again here\n",
    "authors_per_publication = pyscisci.groupby_count(author_publications, colgroupby='PublicationId', \n",
    "                                                 colcountby='AuthorId', \n",
    "                                                   count_unique=True, show_progress=True)\n",
    "# bring back the year info\n",
    "authors_per_publication['Year'] = [pub2year.get(pid, None) for pid in authors_per_publication['PublicationId'].values]\n",
    "\n",
    "# now average\n",
    "number_yearly_authors_a_publication = pyscisci.groupby_mean(authors_per_publication, colgroupby='Year', \n",
    "                                                            colcountby='AuthorIdCount', show_progress=True)\n",
    "\n",
    "number_yearly_authors_a_publication.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.1b - publications per author\n",
    "# first count the number of publications per author per year\n",
    "# (calculated by a groupby on years and count the unique publication ids)\n",
    "\n",
    "# this is known as the yearly productivity\n",
    "publications_per_author_per_year = pyscisci.author_yearly_productivity(author_publications, \n",
    "                                                              colgroupby = 'AuthorId', \n",
    "                                                              datecol = 'Year', \n",
    "                                                              colcountby = 'PublicationId', \n",
    "                                                              show_progress = True)\n",
    "\n",
    "# now average\n",
    "number_yearly_publications_per_author = pyscisci.groupby_mean(publications_per_author_per_year, colgroupby='Year', \n",
    "                                                            colcountby='PublicationIdCount', show_progress=True)\n",
    "\n",
    "number_yearly_publications_per_author.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,2, figsize=(12, 5))\n",
    "\n",
    "ax[0].set_title('a) Overall')\n",
    "\n",
    "ax[0].plot(number_yearly_publications['Year'].values, \n",
    "           number_yearly_publications['PublicationIdCount'].values,\n",
    "          color=red_color, ls='-', label='#papers')\n",
    "\n",
    "ax[0].plot(number_yearly_authors['Year'].values, \n",
    "           number_yearly_authors['AuthorIdCount'].values,\n",
    "          color=lightblue_color, ls='--', label='#authors')\n",
    "\n",
    "ax[0].set_xlim([1900, 2020])\n",
    "ax[0].set_xlabel('Year')\n",
    "ax[0].set_ylabel('Overall number')\n",
    "ax[0].set_yscale('log')\n",
    "\n",
    "\n",
    "ax[1].set_title('b) Average')\n",
    "\n",
    "ax[1].plot(number_yearly_authors_a_publication['Year'].values, \n",
    "           number_yearly_authors_a_publication['PublicationIdCount'].values,\n",
    "          color=red_color, ls='-', label='#authors/papers')\n",
    "\n",
    "ax[1].plot(number_yearly_publications_per_author['Year'].values, \n",
    "           number_yearly_publications_per_author['AuthorIdCount'].values,\n",
    "          color=lightblue_color, ls='--', label='#papers/authors')\n",
    "\n",
    "ax[1].set_xlim([1900, 2020])\n",
    "ax[1].set_xlabel('Year')\n",
    "ax[1].set_ylim([0,5])\n",
    "ax[1].set_ylabel('Average number')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure 1.2 Productivity Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.2a - raw distribution\n",
    "# count the number of publications per author\n",
    "# this is known as the productivity\n",
    "number_publications_per_author = pyscisci.author_productivity(author_publications, \n",
    "                                                              colgroupby = 'AuthorId', \n",
    "                                                              colcountby = 'PublicationId', \n",
    "                                                              show_progress=True)\n",
    "\n",
    "\n",
    "# then we need the number of authors that have the same number of publications\n",
    "productivity_values, author_counts = np.unique(number_publications_per_author['Productivity'].values, \n",
    "                                               return_counts=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# next we can try fitting the lognormal distribution to the raw productivity data \n",
    "# (see cell 2 for def of fit_lognorm)\n",
    "lnrv_s, lnrv_loc, lnrv_scale, lnrv_llh = fit_lognorm(number_publications_per_author['Productivity'].values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1,1, figsize=(6, 5))\n",
    "\n",
    "\n",
    "ax.scatter(productivity_values, \n",
    "           author_counts,\n",
    "          color=lightblue_color, marker='s', label='all authors')\n",
    "\n",
    "\n",
    "lognorm_xs = productivity_values\n",
    "lognorm_ys = author_counts.sum()*spstats.lognorm(lnrv_s, lnrv_loc, lnrv_scale).pdf(lognorm_xs)\n",
    "\n",
    "ax.plot(lognorm_xs, lognorm_ys,\n",
    "          color=red_color, ls='-', label='log-norm. distr.')\n",
    "\n",
    "ax.set_xscale('log')\n",
    "ax.set_xlabel('Number of publications x')\n",
    "ax.set_ylabel('N(x) number of authors with publications')\n",
    "ax.set_yscale('log')\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
