{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "import scipy.sparse as spsparse\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "\n",
    "import pyscisci.all as pyscisci\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set this path to where the MAG database is stored\n",
    "path2mag = '/home/ajgates/MAG'\n",
    "path2mag = '/users/hgt6rn/Documents/DataSets/MAG'\n",
    "\n",
    "# apply a global filter to keep only journal articles\n",
    "docfilter = pyscisci.DocTypeFilter(doctypes=['j'])\n",
    "\n",
    "mymag = pyscisci.MAG(path2mag, database_extension='hdf', keep_in_memory=False,\n",
    "                     global_filter = docfilter, show_progress=True) \n",
    "# set keep_in_memory=False if you want to load the database each time its needed \n",
    "# otherwise keep_in_memory=True will keep each database in memory after its loaded\n",
    "\n",
    "# global_filter will only load publications that match the filtering criteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we will focus on a seminal paper that both measures and embodies interdisciplinarity:\n",
    "\n",
    "# A general framework for analysing diversity in science, technology and society\n",
    "# Andrew Stirling\n",
    "\n",
    "focus_publication_id = 2110158660\n",
    "\n",
    "\n",
    "# the idea here is to understand the interdisciplinary impact of the focus publication \n",
    "\n",
    "# to do this, we take all publications that cite the focus publication, and ask how these are related to eachother\n",
    "# by making the co-citation network\n",
    "# clusters/communities in this network reflect different home disciplines of the publications and \n",
    "# reflect the interdisciplinary nature of our original focus publication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d770a7174434d4ca63577f5c9499e6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa79dc79c8fb4c778ff4783d25a557f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7403 citing publications produce 297 co-cited publications\n"
     ]
    }
   ],
   "source": [
    "# First we load all of the publications citing our focus publication\n",
    "filter_dict = {'CitedPublicationId':np.sort([focus_publication_id])}\n",
    "citing_focus = mymag.load_references(filter_dict=filter_dict)\n",
    "\n",
    "focus_citing_pubs = np.sort(citing_focus['CitingPublicationId'].unique())\n",
    "\n",
    "# Then we need all of the publications that cite these publications\n",
    "pub2ref = mymag.load_references(filter_dict={'CitedPublicationId':focus_citing_pubs})\n",
    "\n",
    "print(\"{0} citing publications produce {1} co-cited publications\".format(pub2ref['CitingPublicationId'].nunique(), \n",
    "                                                                         pub2ref['CitedPublicationId'].nunique()) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d21ae82fd6f8429a83d585c0a507ade3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Fields:   0%|          | 0/151 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# now lets get the field information\n",
    "\n",
    "finfo = mymag.fieldinfo\n",
    "\n",
    "# we'll take the fields at level 0\n",
    "lvl0fields = finfo.loc[finfo['FieldLevel'] == 0][['FieldId', 'FieldName']]\n",
    "fid2name = {fid:fname for fid, fname in lvl0fields.values}\n",
    "\n",
    "\n",
    "# load the mapping of the publication to the fields\n",
    "\n",
    "# we filter only for the publications (co-cited publications above)\n",
    "# and level 0 fields\n",
    "pub2field = mymag.load_pub2field(filter_dict={'PublicationId':np.sort(pub2ref['CitedPublicationId'].unique()), \n",
    "                                              'FieldId':np.sort(lvl0fields['FieldId'].unique())})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CitingPublicationId    6907\n",
       "CitedPublicationId      280\n",
       "dtype: int64"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubs_with_field = np.sort(pub2field['PublicationId'].values)\n",
    "pub2ref = pub2ref.loc[pyscisci.isin_sorted(pub2ref['CitedPublicationId'].values, pubs_with_field)].copy()\n",
    "\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "280 nodes, and 2812 edges\n"
     ]
    }
   ],
   "source": [
    "# we are going to make the co-citation network between all of the publications that cited our focus publication\n",
    "cocitenet, cocite2int = pyscisci.cocitation_network(pub2ref, focus_pub_ids=focus_citing_pubs, \n",
    "                                        focus_constraint='cited', \n",
    "                                        temporal=False, show_progress=False)\n",
    "\n",
    "# cocitenet is our network adjcency matrix\n",
    "# cocite2int maps the publication ids to the row/col of the adjcency matrix\n",
    "print(\"{0} nodes, and {1} edges\".format(len(cocite2int), cocitenet.nnz))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1290\n"
     ]
    }
   ],
   "source": [
    "# elminate self-loops\n",
    "mat = cocitenet.copy()\n",
    "mat.setdiag(0)\n",
    "\n",
    "mat.data[mat.data <=1] = 0\n",
    "mat.eliminate_zeros()\n",
    "\n",
    "print(mat.nnz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "153\n",
      "123\n"
     ]
    }
   ],
   "source": [
    "n_components, labels = spsparse.csgraph.connected_components(mat)\n",
    "print(n_components)\n",
    "comidx, compsizes = np.unique(labels, return_counts=True)\n",
    "print(max(compsizes))\n",
    "\n",
    "lccidx = np.argmax(compsizes)\n",
    "\n",
    "lcc = labels==lccidx\n",
    "lccnodes = np.arange(mat.shape[0])[lcc]\n",
    "remapnodes = {nid:i for i, nid in enumerate(lccnodes)}\n",
    "cocite2int = {pid:remapnodes[i] for pid, i in cocite2int.items() if not remapnodes.get(i, None) is None}\n",
    "\n",
    "lccmat = spsparse.coo_matrix(spsparse.csr_matrix(mat)[lcc][:,lcc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PublicationId    123\n",
       "FieldId           12\n",
       "dtype: int64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cocite_pubs = np.sort(list(cocite2int.keys()))\n",
    "cocite_fields = pub2field.loc[pyscisci.isin_sorted(pub2field['PublicationId'].values, cocite_pubs)].copy()\n",
    "\n",
    "cocite_fields.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41008148 computer science 33\n",
      "144024400 sociology 32\n",
      "162324750 economics 20\n",
      "144133560 business 12\n",
      "33923547 mathematics 7\n",
      "17744445 political science 6\n",
      "205649164 geography 3\n",
      "15744967 psychology 3\n",
      "71924100 medicine 2\n",
      "127413603 engineering 2\n",
      "121332964 physics 2\n",
      "86803240 biology 1\n"
     ]
    }
   ],
   "source": [
    "# most publications are mapped to multiple fields, even at level 1 \n",
    "# to color our nodes, lets start by seeing which fields appear the most frequently\n",
    "fieldcount = cocite_fields['FieldId'].value_counts(ascending=False)\n",
    "\n",
    "for idx in fieldcount.index.values:\n",
    "    print(idx, fid2name[idx], fieldcount.get(idx))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we are going to take the largest 3 fields, and group together all of the remaining fields into an \"other\"\n",
    "max_fields = 3\n",
    "reorder_fields = {fid:i for i, fid in enumerate(fieldcount.index.values[:(max_fields+1)])}\n",
    "\n",
    "cocite_fields['RemappedFields'] = [reorder_fields.get(fid, max_fields) for fid in cocite_fields['FieldId'].values]\n",
    "cocite_fields.sort_values(by=['RemappedFields'], ascending=True, inplace=True)\n",
    "cocite_fields.drop_duplicates(subset=['PublicationId'], keep='first', inplace=True)\n",
    "\n",
    "cocite_fields.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 computer science 33\n",
      "1 sociology 32\n",
      "2 economics 20\n",
      "3 other 38\n"
     ]
    }
   ],
   "source": [
    "fieldcount = cocite_fields['RemappedFields'].value_counts(ascending=False)\n",
    "reorder_fields2 = {i:idx for idx, i in reorder_fields.items()}\n",
    "for idx in fieldcount.index.values[:(max_fields+1)]:\n",
    "    if idx < max_fields:\n",
    "        print(idx, fid2name[reorder_fields2[idx]], fieldcount.get(idx))\n",
    "        \n",
    "print(max_fields, \"other\", fieldcount.get(max_fields))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#field2color = {0:'#DE9E36', 1:'#EF6461',4:'#C287E8',3:'#DFB2F4',2:'#87BFFF'}\n",
    "field2color = {0:'#E9B44C', 1:'#50A2A7',2:'#8E5572', 3:'#5E8334'}\n",
    "field2node = defaultdict(list)\n",
    "\n",
    "for pid, fid in cocite_fields[['PublicationId', 'RemappedFields']].values:\n",
    "    field2node[fid].append(cocite2int[pid])\n",
    "\n",
    "# finally, one publication didnt have a field label\n",
    "#field2node[3].append(96)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = nx.Graph(lccmat)\n",
    "\n",
    "pos = nx.drawing.nx_agraph.graphviz_layout(g, prog='sfdp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 33\n",
      "1 32\n",
      "2 20\n",
      "3 38\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 175860 characters]\n",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,1,figsize=(12,8))         \n",
    "\n",
    "\n",
    "\n",
    "nx.draw_networkx_edges(g, pos, edgelist=g.edges(),\n",
    "        edge_color = '0.8',\n",
    "        alpha=0.8,\n",
    "        ax=ax)\n",
    "\n",
    "for fid in field2node.keys():\n",
    "    print(fid, len(field2node[fid]))\n",
    "    nx.draw_networkx_nodes(g.subgraph(field2node[fid]), pos,\n",
    "        node_size= 100, \n",
    "        node_color=field2color[fid], \n",
    "        edgecolors='0.2', # node outline\n",
    "                       vmax = 5,\n",
    "        ax = ax)\n",
    "    #print(fid, field2node[fid], len(g.subgraph(field2node[fid])))\n",
    "\n",
    "#ax.set_facecolor('#09101F')\n",
    "\n",
    "plt.savefig('StirlingCocitiation.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 4\n",
      "123 123\n",
      "Element-centric similarity: 0.3262145089174398\n",
      "0 0.34856587263004374\n",
      "1 0.30595128676470584\n",
      "2 0.4542827794247672\n",
      "3 0.2564635272391506\n"
     ]
    }
   ],
   "source": [
    "# we use cdlib to conduct community dectection in networks\n",
    "# pip install cdlib\n",
    "from cdlib import algorithms\n",
    "\n",
    "# and then conduct similarity analysis using clusim\n",
    "# pip install clusim\n",
    "from clusim.clustering import Clustering, print_clustering\n",
    "import clusim.sim as sim\n",
    "\n",
    "\n",
    "coms = algorithms.louvain(g, weight='weight', resolution=1., randomize=False)\n",
    "\n",
    "louvain_communities = Clustering().from_cluster_list(coms.communities)\n",
    "field_communities = Clustering(clu2elm_dict = field2node)\n",
    "\n",
    "\n",
    "print(louvain_communities.n_clusters, field_communities.n_clusters)\n",
    "print(louvain_communities.n_elements, field_communities.n_elements)\n",
    "\n",
    "print(\"Element-centric similarity: {}\".format(sim.element_sim(louvain_communities, field_communities, alpha = 0.9)) )\n",
    "\n",
    "elscore, _ = sim.element_sim_elscore(louvain_communities, field_communities, alpha = 0.9)\n",
    "\n",
    "for c, elist in field2node.items():\n",
    "    print(c, elscore[elist].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 177644 characters]\n",
      "text/plain": [
       "<Figure size 1200x1200 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# draw the louvain clusters\n",
    "\n",
    "fig, ax = plt.subplots(1,1,figsize=(12,12))         \n",
    "\n",
    "nx.draw_networkx_edges(g, pos, edgelist=g.edges(),\n",
    "        edge_color = '0.8',\n",
    "        alpha=0.2,\n",
    "        ax=ax)\n",
    "\n",
    "for c in louvain_communities.clusters:\n",
    "    nx.draw_networkx_nodes(g.subgraph(louvain_communities.clu2elm_dict[c]), pos,\n",
    "        node_size= 100, \n",
    "        node_color=field2color[c], \n",
    "        edgecolors='0.2', # node outline\n",
    "                       vmax = 5,\n",
    "        ax = ax)\n",
    "    #print(fid, field2node[c], len(g.subgraph(field2node[c])))\n",
    "\n",
    "#ax.set_facecolor('#09101F')\n",
    "\n",
    "#plt.savefig('DiversityCocitiation.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0137403368935076"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def estimate_resolution(G, com):\n",
    "    \"\"\"\n",
    "    Newman, MEJ (2016) Community detection in networks: Modularity optimization and maximum likelihood are equivalent. Phy. Rev. E\n",
    "    \"\"\"\n",
    "    m = G.number_of_edges()\n",
    "    \n",
    "    # eq 16\n",
    "    kappas = [sum(deg for n, deg in G.degree(c)) for c in com.communities]\n",
    "    m_in = [G.subgraph(c).number_of_edges() for c in com.communities]\n",
    "    \n",
    "    denom = np.sum(np.square(kappas)) / (2*m)\n",
    "    \n",
    "    # eq 17\n",
    "    omega_in = 2*sum(m_in) / denom\n",
    "    \n",
    "    # eq 18\n",
    "    omega_out = (2*m - 2*sum(m_in)) / (2*m - denom)\n",
    "    \n",
    "    # eq 15\n",
    "    gamma = (omega_in - omega_out) / (np.log(omega_in) - np.log(omega_out))\n",
    "    \n",
    "    return gamma\n",
    "\n",
    "estimate_resolution(g, coms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0137403368935076"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coms = algorithms.louvain(g, weight='weight', resolution=1.013, randomize=False)\n",
    "estimate_resolution(g, coms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
