{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f819f8ce",
   "metadata": {},
   "source": [
    "# Comparison of Career Topic Switching from three different Databases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "316c56ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "import pyscisci.all as pyscisci\n",
    "\n",
    "try:\n",
    "    import seaborn as sns\n",
    "    sns.set_style('white')\n",
    "    \n",
    "    from cdlib import algorithms\n",
    "    \n",
    "    from clusim.clustering import Clustering\n",
    "except:\n",
    "    print('This example requires the optional packages seaborn, cdlib and clusim.')\n",
    "    print('Please run:')\n",
    "    print('pip install seaborn cdlib clusim')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc7e160f",
   "metadata": {},
   "source": [
    "Setup the three databases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "054f2fa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set this path to where the APS database is stored\n",
    "path2aps = '/home/ajgates/APS'\n",
    "path2aps = '/Volumes/GatesExpansionDrive/DataSets/APS/APS2019'\n",
    "\n",
    "myaps = pyscisci.APS(path2aps, keep_in_memory=False) \n",
    "\n",
    "# override the default APS data with the author disambiguation\n",
    "myaps.set_new_data_path('paa', 'publicationauthoraffiliation_supp2010')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "75b097eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set this path to where the MAG database is stored\n",
    "path2mag = '/home/ajgates/MAG'\n",
    "path2mag = '/Volumes/GatesExpansionDrive/DataSets/MAG'\n",
    "\n",
    "mymag = pyscisci.MAG(path2mag, show_progress=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3035eb0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set this path to where the MAG database is stored\n",
    "path2wos = '/home/ajgates/WOS'\n",
    "\n",
    "mywos = pyscisci.WOS(path2wos, show_progress=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5ac23fcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we pick a prominent physics author to study\n",
    "\n",
    "mag_author_id = 2420755856\n",
    "aps_author_id = 81060\n",
    "wos_author_id = 605000\n",
    "\n",
    "three_databases = [('APS', myaps, aps_author_id), ('MAG', mymag, mag_author_id)] #,('WOS', mywos, wos_author_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0dbfeaaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90499ece3df9481eb052244ccd897a68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publication Author Affiliation:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcc6fbf074364ad0badfc02f3514793e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "APS: Graph with 302 nodes and 2955 edges\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "734a3a3188cd4d70b0c486733b3b21f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publication Author Affiliation:   0%|          | 0/63 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "992f501b9eae4c58a70d2c508d2a3e86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/160 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAG: Graph with 452 nodes and 10440 edges\n",
      "WOS: Graph with 881 nodes and 37224 edges\n"
     ]
    }
   ],
   "source": [
    "author_networks = []\n",
    "\n",
    "for dbname, database, author_id in three_databases:\n",
    "    \n",
    "    # we load the publication-author relationships the same way from each database\n",
    "    author_filter_dict = {'AuthorId':np.sort([author_id])}\n",
    "    author_paa = database.load_publicationauthoraffiliation(columns=['PublicationId', 'AuthorId'],\n",
    "                                                            filter_dict=author_filter_dict, \n",
    "                                                            duplicate_subset=['PublicationId', 'AuthorId'])\n",
    "    \n",
    "    # because the aps author info is only until 2010, we standardize across all three databases\n",
    "    pub2year = database.pub2year\n",
    "    author_paa['Year'] = [pub2year.get(pid, None) for pid in author_paa['PublicationId'].values]\n",
    "    author_paa = author_paa[author_paa['Year'] <= 2010]\n",
    "    \n",
    "    # now load the publication to reference linkages only for the authors publications\n",
    "    pub_filter_dict = {'CitingPublicationId':np.sort(author_paa['PublicationId'].unique())}\n",
    "    pub2ref = database.load_references(columns=['CitingPublicationId', 'CitedPublicationId'],\n",
    "                                                            filter_dict=pub_filter_dict)\n",
    "    \n",
    "    career_topics, cocite_network = pyscisci.career_cociting_network_topics(author_paa, \n",
    "                                                                        pub2ref, \n",
    "                                        randomize=42, \n",
    "                                        return_network=True, show_progress=False)\n",
    "    author_networks.append([dbname, career_topics, cocite_network])\n",
    "    print(dbname + ':', nx.info(cocite_network))\n",
    "    \n",
    "author_paa = pd.read_hdf('/Users/ajgates/Desktop/wos_paa_stanley.hdf')    \n",
    "pub2ref = pd.read_hdf('/Users/ajgates/Desktop/wos_ref_stanley.hdf')\n",
    "\n",
    "author_paa = author_paa[author_paa['PubYear'] <= 2010]\n",
    "author_paa = author_paa.rename(columns={'ArticleID':'PublicationId', 'PubYear':'Year', \n",
    "                                        'AuthorDAIS': 'AuthorId'})\n",
    "pub2ref = pub2ref.rename(columns={'CitingArticleID':'CitingPublicationId', \n",
    "                                  'CitedArticleID':'CitedPublicationId'})\n",
    "\n",
    "career_topics, cocite_network = pyscisci.career_cociting_network_topics(author_paa, \n",
    "                                                                        pub2ref, \n",
    "                                        randomize=42, \n",
    "                                        return_network=True, show_progress=False)\n",
    "author_networks.append(['WOS', career_topics, cocite_network])\n",
    "print('WOS' + ':', nx.info(cocite_network))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3e285f3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(102007395, 7)\n"
     ]
    }
   ],
   "source": [
    "afiles = [fname for fname in os.listdir('/Volumes/GatesExpansionDrive/DataSets/WOSold/author/') if '.hdf' in fname]\n",
    "allauthors = []\n",
    "for fname in afiles:\n",
    "    allauthors.append(pd.read_hdf('/Volumes/GatesExpansionDrive/DataSets/WOSold/author/' + fname).dropna(subset=['AuthorDAIS']))\n",
    "\n",
    "allauthors = pd.concat(allauthors)\n",
    "print(allauthors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2119ea3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1044, 22)\n"
     ]
    }
   ],
   "source": [
    "afiles = [fname for fname in os.listdir('/Volumes/GatesExpansionDrive/DataSets/WOSold/article/') if '.hdf' in fname]\n",
    "allpubs = []\n",
    "for fname in afiles:\n",
    "    pdf = pd.read_hdf('/Volumes/GatesExpansionDrive/DataSets/WOSold/article/' + fname)\n",
    "    allpubs.append(pdf[pyscisci.isin_sorted(pdf['ArticleID'].values, np.sort(wos_paa['ArticleID'].unique()))])\n",
    "allpubs = pd.concat(allpubs)\n",
    "print(allpubs.shape)\n",
    "\n",
    "wos_paa= allauthors[allauthors['AuthorDAIS']==605000.0].reset_index(drop=True)\n",
    "wos_paa = wos_paa.merge(allpubs, how='left', on='ArticleID')\n",
    "wos_paa.to_hdf('/Users/ajgates/Desktop/wos_paa_stanley.hdf', key='paa', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "cdfe3d45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(38302, 2)\n"
     ]
    }
   ],
   "source": [
    "afiles = [fname for fname in os.listdir('/Volumes/GatesExpansionDrive/DataSets/WOSold/citation/') if '.hdf' in fname]\n",
    "allrefs = []\n",
    "for fname in afiles:\n",
    "    cdf = pd.read_hdf('/Volumes/GatesExpansionDrive/DataSets/WOSold/citation/' + fname)\n",
    "    allrefs.append(cdf[pyscisci.isin_sorted(cdf['CitingArticleID'].values, np.sort(wos_paa['ArticleID'].unique()))])\n",
    "allrefs = pd.concat(allrefs)\n",
    "print(allrefs.shape)\n",
    "\n",
    "allrefs.to_hdf('/Users/ajgates/Desktop/wos_ref_stanley.hdf', key='ref', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bd9293f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 162852 characters]\n",
      "text/plain": [
       "<Figure size 864x648 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(3,2, figsize=(12, 9))\n",
    "\n",
    "cmap = sns.color_palette(\"Set2\", as_cmap=True)\n",
    "\n",
    "for iax in range(3):\n",
    "\n",
    "    dbname, career_topics, cocite_network = author_networks[iax]\n",
    "    \n",
    "    pos = nx.drawing.nx_agraph.graphviz_layout(cocite_network, prog='sfdp',)\n",
    "  \n",
    "\n",
    "    nx.draw_networkx_edges(cocite_network, pos, edgelist=cocite_network.edges(),\n",
    "            edge_color = '0.7', width=0.3,\n",
    "            ax=ax[iax,0])\n",
    "    \n",
    "    topic_community = nx.get_node_attributes(cocite_network, 'TopicCommunity')\n",
    "\n",
    "    louvain_communities = Clustering().from_membership_list([topic_community[i] for i in range(len(topic_community))])\n",
    "\n",
    "    for c in louvain_communities.clusters:\n",
    "        nx.draw_networkx_nodes(cocite_network.subgraph(louvain_communities.clu2elm_dict[c]), pos,\n",
    "            node_size= 20, \n",
    "            node_color=[c]*len(louvain_communities.clu2elm_dict[c]),\n",
    "            cmap = cmap,\n",
    "            #edgecolors='0.2', # node outline\n",
    "            linewidths = None,\n",
    "            vmin = 0,\n",
    "            vmax = louvain_communities.n_clusters,\n",
    "            label = str(c),\n",
    "            ax = ax[iax,0])\n",
    "\n",
    "    \n",
    "    \n",
    "    for y, topic, degree in career_topics[['Year', 'TopicCommunity', 'Degree']].values:\n",
    "        ax[iax,1].scatter(y, degree, c=[topic], cmap=cmap, vmin=0,\n",
    "                   vmax=louvain_communities.n_clusters, zorder=5)\n",
    "        ax[iax,1].plot([y,y], [0,degree], color='k', lw=0.25, zorder=4)\n",
    "\n",
    "plt.savefig('DatabaseCompareTopicSwitching.pdf')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6871171f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5c7a385d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2af0c0a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
