{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8811692c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import gzip\n",
    "import itertools\n",
    "from itertools import chain\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import networkx as nx\n",
    "\n",
    "import scipy.sparse as spsparse\n",
    "import scipy.stats as spstats\n",
    "\n",
    "from statsmodels.stats.multitest import multipletests as holm\n",
    "import statsmodels.api as sm\n",
    "\n",
    "from linearmodels import PanelOLS\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "import matplotlib.transforms as mtransforms\n",
    "import seaborn as sns\n",
    "\n",
    "import pyscisci.all as pyscisci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c11ab19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate the AUC for each country pair\n",
    "\n",
    "\n",
    "# read in the country citation data\n",
    "for dataset in [\"oa_countrycites\", \"oa_countrycites_noself\", \"oa_countrycites_nosameorg\"]:\n",
    "\n",
    "    countrycites = pd.read_csv(os.path.join(path2countrydata, dataset + \".csv.gz\"))\n",
    "\n",
    "    country_auc_df = []\n",
    "    def country_auc(citedf):\n",
    "        for cited_c in citedf['Country'].unique():\n",
    "            x = citedf[citedf['Country'] == cited_c]['CountryCitations'].values\n",
    "            y = citedf[citedf['Country'] != cited_c]['CountryCitations'].values\n",
    "            auc, delongcov = pyscisci.fast_delong(x, y)\n",
    "            cname, y = citedf.name\n",
    "            country_auc_df.append([cname, y, cited_c, auc, delongcov, x.shape[0]])\n",
    "\n",
    "    countrycites.groupby(['CitingCountry', 'CitedYear'])[['Country', 'CountryCitations']].apply(country_auc)\n",
    "    country_auc_df = pd.DataFrame(country_auc_df, columns=['CitingCountry', 'CitedYear', 'CitedCountry', 'AUC', 'Cov', 'N'])\n",
    "\n",
    "    country_auc_df.to_csv(os.path.join(path2countrydata, dataset + '_auc.csv.gz'), compression='gzip', \n",
    "                                       index=False, header=True, mode='w')\n",
    "\n",
    "    print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eefca44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets focus on the citation without the same organization and self citations\n",
    "\n",
    "country_auc_df = pd.read_csv(os.path.join('CountryData', 'oa_countrycites_nosameorg_auc.csv.gz'))\n",
    "\n",
    "country_auc_df = country_auc_df[country_auc_df['N']>=50]\n",
    "\n",
    "country_auc_df['Zscore'] = (country_auc_df['AUC'] - 0.5)/np.sqrt(country_auc_df['Cov'])\n",
    "country_auc_df['pvalue'] = spstats.norm.sf(np.abs(country_auc_df['Zscore'].values))\n",
    "\n",
    "def reject_holm(pvalues):\n",
    "    reject, pvalues, _, _ = holm(pvalues, alpha=0.01, method='hs', is_sorted=False, returnsorted=False)\n",
    "    return reject\n",
    "country_auc_df['Significant'] = country_auc_df.groupby('CitedYear')['pvalue'].transform(reject_holm)\n",
    "\n",
    "net_auc = country_auc_df[country_auc_df['CitingCountry'] != country_auc_df['CitedCountry']].sort_values(by=['CitedYear', 'CitedCountry']).reset_index(drop=True)\n",
    "net_auc = net_auc[net_auc['Significant']]\n",
    "self_auc = country_auc_df[country_auc_df['CitingCountry'] == country_auc_df['CitedCountry']].sort_values(by=['CitedYear', 'CitedCountry']).reset_index(drop=True)\n",
    "self_auc = self_auc[self_auc['Significant']]\n",
    "\n",
    "comm2country = {0:['AE', 'BD', 'CN', 'CY', 'DZ', 'EG', 'ID', 'IN', 'IQ', 'IR', 'JO', 'KH', 'KR', 'KW', 'MA', 'MY', 'OM', 'PK', 'QA', 'SA', 'SG', 'TH', 'TN', 'TR', 'TW', 'VN'],\n",
    "1:['AR', 'BN', 'BO', 'BR', 'BW', 'BY', 'CL', 'CM', 'CO', 'CR', 'GE', 'HK', 'HR', 'LU', 'MZ', 'NC', 'PA', 'PE', 'RU', 'SJ', 'ST', 'UY', 'VE'],\n",
    "2:['AT', 'BE', 'CA', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GR', 'HU', 'IE', 'IL', 'IS', 'IT', 'JP', 'MX', 'NL', 'NO', 'NP', 'NZ', 'PH', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK'],\n",
    "3:['AU', 'CH', 'GB', 'US'],\n",
    "4:['BH', 'ET', 'GH', 'KE', 'MW', 'NG', 'TZ', 'UG', 'ZA', 'ZM', 'ZW']}\n",
    "\n",
    "country2comm = {c:ci for ci,clist in comm2country.items() for c in clist}\n",
    "poscountrylist = sorted(list(country2comm.keys()))\n",
    "\n",
    "connected_countries = np.sort(np.unique(net_auc[net_auc['Zscore'] > 0][['CitingCountry', 'CitedCountry']].values.flatten()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "aa879d8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5869, 5)\n"
     ]
    }
   ],
   "source": [
    "countryprod = pd.read_csv('CountryData/CountryTopProductivity.csv')\n",
    "syear = 1990\n",
    "eyear = 2017\n",
    "\n",
    "countryprod = countryprod[countryprod['Year'] >= syear]\n",
    "countryprod = countryprod[countryprod['Year'] <= eyear].reset_index(drop=True)\n",
    "print(countryprod.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "69ea2fc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Country</th>\n",
       "      <th>NumPub</th>\n",
       "      <th>TopJournal</th>\n",
       "      <th>FracTop</th>\n",
       "      <th>PositiveInDegree</th>\n",
       "      <th>NegativeInDegree</th>\n",
       "      <th>PositiveOutDegree</th>\n",
       "      <th>NegativeOutDegree</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AD</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AE</td>\n",
       "      <td>109</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.669725</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AF</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AL</td>\n",
       "      <td>34</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.294118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AM</td>\n",
       "      <td>212</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.452830</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5864</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>XK</td>\n",
       "      <td>631</td>\n",
       "      <td>84.0</td>\n",
       "      <td>0.133122</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5865</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>YE</td>\n",
       "      <td>403</td>\n",
       "      <td>106.0</td>\n",
       "      <td>0.263027</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5866</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>ZA</td>\n",
       "      <td>23661</td>\n",
       "      <td>8606.0</td>\n",
       "      <td>0.363721</td>\n",
       "      <td>11.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5867</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>ZM</td>\n",
       "      <td>702</td>\n",
       "      <td>306.0</td>\n",
       "      <td>0.435897</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5868</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>ZW</td>\n",
       "      <td>1103</td>\n",
       "      <td>324.0</td>\n",
       "      <td>0.293744</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5869 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Year Country  NumPub  TopJournal   FracTop  PositiveInDegree  \\\n",
       "0     1990.0      AD       1         0.0  0.000000               0.0   \n",
       "1     1990.0      AE     109        73.0  0.669725               0.0   \n",
       "2     1990.0      AF       2         1.0  0.500000               0.0   \n",
       "3     1990.0      AL      34        10.0  0.294118               0.0   \n",
       "4     1990.0      AM     212        96.0  0.452830               0.0   \n",
       "...      ...     ...     ...         ...       ...               ...   \n",
       "5864  2017.0      XK     631        84.0  0.133122               0.0   \n",
       "5865  2017.0      YE     403       106.0  0.263027               0.0   \n",
       "5866  2017.0      ZA   23661      8606.0  0.363721              11.0   \n",
       "5867  2017.0      ZM     702       306.0  0.435897               1.0   \n",
       "5868  2017.0      ZW    1103       324.0  0.293744               2.0   \n",
       "\n",
       "      NegativeInDegree  PositiveOutDegree  NegativeOutDegree  \n",
       "0                  0.0                0.0                0.0  \n",
       "1                  0.0                0.0                0.0  \n",
       "2                  0.0                0.0                0.0  \n",
       "3                  0.0                0.0                0.0  \n",
       "4                  0.0                0.0                1.0  \n",
       "...                ...                ...                ...  \n",
       "5864               2.0                0.0                2.0  \n",
       "5865               1.0                0.0                0.0  \n",
       "5866              12.0               14.0               16.0  \n",
       "5867               2.0                1.0                4.0  \n",
       "5868               5.0                1.0                4.0  \n",
       "\n",
       "[5869 rows x 9 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pin = []\n",
    "nin = []\n",
    "pout = []\n",
    "nout = []\n",
    "\n",
    "for y in np.arange(syear, eyear + 1):\n",
    "    ysub = net_auc[net_auc['CitedYear'] <= y].drop_duplicates(subset=['CitingCountry', 'CitedCountry'], keep='last')\n",
    "    \n",
    "    ypin = ysub[ysub['Zscore'] > 0].groupby('CitedCountry', as_index=False)['CitingCountry'].nunique()\n",
    "    ypin = ypin.rename(columns={'CitedCountry':'Country', 'CitingCountry':'PositiveInDegree'})\n",
    "    ypin['Year'] = y\n",
    "    pin.append(ypin)\n",
    "    \n",
    "    ynin = ysub[ysub['Zscore'] < 0].groupby('CitedCountry', as_index=False)['CitingCountry'].nunique()\n",
    "    ynin = ynin.rename(columns={'CitedCountry':'Country', 'CitingCountry':'NegativeInDegree'})\n",
    "    ynin['Year'] = y\n",
    "    nin.append(ynin)\n",
    "    \n",
    "    ypout = ysub[ysub['Zscore'] > 0].groupby('CitingCountry', as_index=False)['CitedCountry'].nunique()\n",
    "    ypout = ypout.rename(columns={'CitingCountry':'Country', 'CitedCountry':'PositiveOutDegree'})\n",
    "    ypout['Year'] = y\n",
    "    pout.append(ypout)\n",
    "    \n",
    "    ynout = ysub[ysub['Zscore'] < 0].groupby('CitingCountry', as_index=False)['CitedCountry'].nunique()\n",
    "    ynout = ynout.rename(columns={'CitingCountry':'Country', 'CitedCountry':'NegativeOutDegree'})\n",
    "    ynout['Year'] = y\n",
    "    nout.append(ynout)\n",
    "    \n",
    "\n",
    "pin = pd.concat(pin)    \n",
    "countryprod = countryprod.merge(pin, how='left', on=['Country', 'Year']).fillna(0)\n",
    "nin = pd.concat(nin)    \n",
    "countryprod = countryprod.merge(nin, how='left', on=['Country', 'Year']).fillna(0)\n",
    "pout = pd.concat(pout)    \n",
    "countryprod = countryprod.merge(pout, how='left', on=['Country', 'Year']).fillna(0)\n",
    "nout = pd.concat(nout)    \n",
    "countryprod = countryprod.merge(nout, how='left', on=['Country', 'Year']).fillna(0)\n",
    "countryprod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "55489bdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Country</th>\n",
       "      <th>NumPub</th>\n",
       "      <th>TopJournal</th>\n",
       "      <th>FracTop</th>\n",
       "      <th>PositiveInDegree</th>\n",
       "      <th>NegativeInDegree</th>\n",
       "      <th>PositiveOutDegree</th>\n",
       "      <th>NegativeOutDegree</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1789</th>\n",
       "      <td>1998.0</td>\n",
       "      <td>US</td>\n",
       "      <td>288415</td>\n",
       "      <td>188631.0</td>\n",
       "      <td>0.654026</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>64.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994</th>\n",
       "      <td>1999.0</td>\n",
       "      <td>US</td>\n",
       "      <td>296694</td>\n",
       "      <td>190399.0</td>\n",
       "      <td>0.641735</td>\n",
       "      <td>18.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>69.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3938</th>\n",
       "      <td>2009.0</td>\n",
       "      <td>CH</td>\n",
       "      <td>26245</td>\n",
       "      <td>15715.0</td>\n",
       "      <td>0.598781</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4399</th>\n",
       "      <td>2011.0</td>\n",
       "      <td>GB</td>\n",
       "      <td>153745</td>\n",
       "      <td>75751.0</td>\n",
       "      <td>0.492705</td>\n",
       "      <td>18.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>53.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4619</th>\n",
       "      <td>2012.0</td>\n",
       "      <td>GB</td>\n",
       "      <td>160714</td>\n",
       "      <td>83316.0</td>\n",
       "      <td>0.518412</td>\n",
       "      <td>18.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>57.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4817</th>\n",
       "      <td>2013.0</td>\n",
       "      <td>DE</td>\n",
       "      <td>136903</td>\n",
       "      <td>68633.0</td>\n",
       "      <td>0.501326</td>\n",
       "      <td>18.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>59.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4973</th>\n",
       "      <td>2013.0</td>\n",
       "      <td>US</td>\n",
       "      <td>623646</td>\n",
       "      <td>318679.0</td>\n",
       "      <td>0.510993</td>\n",
       "      <td>18.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>86.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5193</th>\n",
       "      <td>2014.0</td>\n",
       "      <td>US</td>\n",
       "      <td>643283</td>\n",
       "      <td>315858.0</td>\n",
       "      <td>0.491009</td>\n",
       "      <td>18.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>87.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5225</th>\n",
       "      <td>2015.0</td>\n",
       "      <td>BE</td>\n",
       "      <td>30030</td>\n",
       "      <td>15760.0</td>\n",
       "      <td>0.524809</td>\n",
       "      <td>18.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5479</th>\n",
       "      <td>2016.0</td>\n",
       "      <td>DK</td>\n",
       "      <td>26103</td>\n",
       "      <td>14461.0</td>\n",
       "      <td>0.553998</td>\n",
       "      <td>18.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Year Country  NumPub  TopJournal   FracTop  PositiveInDegree  \\\n",
       "1789  1998.0      US  288415    188631.0  0.654026              18.0   \n",
       "1994  1999.0      US  296694    190399.0  0.641735              18.0   \n",
       "3938  2009.0      CH   26245     15715.0  0.598781              18.0   \n",
       "4399  2011.0      GB  153745     75751.0  0.492705              18.0   \n",
       "4619  2012.0      GB  160714     83316.0  0.518412              18.0   \n",
       "4817  2013.0      DE  136903     68633.0  0.501326              18.0   \n",
       "4973  2013.0      US  623646    318679.0  0.510993              18.0   \n",
       "5193  2014.0      US  643283    315858.0  0.491009              18.0   \n",
       "5225  2015.0      BE   30030     15760.0  0.524809              18.0   \n",
       "5479  2016.0      DK   26103     14461.0  0.553998              18.0   \n",
       "\n",
       "      NegativeInDegree  PositiveOutDegree  NegativeOutDegree  \n",
       "1789               1.0                3.0               64.0  \n",
       "1994               4.0                3.0               69.0  \n",
       "3938               0.0                4.0               25.0  \n",
       "4399               9.0               15.0               53.0  \n",
       "4619               9.0               17.0               57.0  \n",
       "4817              19.0               11.0               59.0  \n",
       "4973              28.0               14.0               86.0  \n",
       "5193              29.0               18.0               87.0  \n",
       "5225               3.0                9.0               18.0  \n",
       "5479              10.0               10.0               16.0  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countryprod[countryprod['PositiveInDegree'] == 18]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4f7dbfe8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ysub = net_auc[net_auc['CitedYear'] <= 2017].drop_duplicates(subset=['CitingCountry', 'CitedCountry'], keep='last')\n",
    "ysub['Weight'] = np.sign(ysub['Zscore'])\n",
    "ysub = ysub.rename(columns={'CitingCountry':'Source', 'CitedCountry':'Target'})\n",
    "ysub[ysub['Zscore'] < 0][['Source', 'Target', 'Weight']].to_csv('CountryNetworks/NegativeEdges_12_23.csv', index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "d9a4b1d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "syear, eyear = 1990, 2017\n",
    "\n",
    "ygini = np.zeros((eyear-syear + 1))\n",
    "\n",
    "for y in range(syear, eyear + 1):\n",
    "    \n",
    "    ysub = net_auc[net_auc['CitedYear'] <= y].drop_duplicates(subset=['CitingCountry', 'CitedCountry'], keep='last')\n",
    "    posnet = ysub[ysub['Zscore']>0]\n",
    "\n",
    "    usednodes = np.sort(np.unique(posnet[['CitingCountry', 'CitedCountry']].values.flatten()))\n",
    "    \n",
    "    ypnet = nx.DiGraph()\n",
    "    ypnet.add_nodes_from(usednodes)\n",
    "    ypnet.add_edges_from([(s, t) for s,t in posnet[['CitingCountry', 'CitedCountry']].values])\n",
    "    \n",
    "    prdict = nx.pagerank(ypnet, alpha=0.85)\n",
    "    pr = np.array([prdict[v] for v in ypnet.nodes()])\n",
    "    #ygini[y-syear] = gini(pr)\n",
    "    ygini[y-syear] = spstats.entropy(pr/pr.sum())/np.log2(len(ypnet))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "cb2fb15a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4] [ 4 29 26 11 23]\n"
     ]
    }
   ],
   "source": [
    "syear, eyear = 1990, 2017\n",
    "\n",
    "renumber_blocks = {3:0, 2:1,1:4,0:2,4:3}\n",
    "\n",
    "v2com = np.array([renumber_blocks[country2comm[c]] for c in poscountrylist])\n",
    "\n",
    "fullcnames, fullcsizes = np.unique(v2com, return_counts=True)\n",
    "print(fullcnames, fullcsizes)\n",
    "nc = fullcnames.shape[0]\n",
    "\n",
    "pblock_matrix = np.zeros((eyear - syear + 1, nc,nc))\n",
    "nblock_matrix = np.zeros((eyear - syear + 1, nc,nc))\n",
    "\n",
    "for y in range(syear, eyear + 1):\n",
    "    \n",
    "    ysub = net_auc[net_auc['CitedYear'] <= y].drop_duplicates(subset=['CitingCountry', 'CitedCountry'], keep='last')\n",
    "    posnet = ysub[ysub['Zscore']>0]\n",
    "    \n",
    "    usednodes = np.sort(np.unique(posnet[['CitingCountry', 'CitedCountry']].values.flatten()))\n",
    "    yv2com = v2com[pyscisci.isin_sorted(connected_countries, usednodes)]\n",
    "    \n",
    "    pnet = nx.DiGraph()\n",
    "    pnet.add_nodes_from(usednodes)\n",
    "    pnet.add_edges_from([(s, t) for s,t in posnet[['CitingCountry', 'CitedCountry']].values])\n",
    "    padjmat = nx.to_numpy_array(pnet, nodelist = usednodes)\n",
    "    \n",
    "    negnet = ysub[ysub['Zscore']<0]\n",
    "    nnet = nx.DiGraph()\n",
    "    nnet.add_nodes_from(usednodes)\n",
    "    nnet.add_edges_from([(s, t) for s,t in negnet[['CitingCountry', 'CitedCountry']].values])\n",
    "    nadjmat = nx.to_numpy_array(nnet, nodelist = usednodes)\n",
    "    \n",
    "    cnames, csizes = np.unique(yv2com, return_counts=True)\n",
    "    csizes = dict(zip(cnames, csizes))\n",
    "    #print(y, yv2com.shape, padjmat.shape, nadjmat.shape)\n",
    "    #print(cnames, csizes )\n",
    "    \n",
    "    for ic in cnames:\n",
    "        for jc in cnames:\n",
    "            #denom = csizes[ic]*csizes[jc]\n",
    "            denom = fullcsizes[ic]*fullcsizes[jc]\n",
    "            pblock_matrix[y-syear, ic, jc] = padjmat[yv2com==ic][:,yv2com==jc].sum()/denom\n",
    "            nblock_matrix[y-syear, ic, jc] = nadjmat[yv2com==ic][:,yv2com==jc].sum()/denom\n",
    "\n",
    "    #print(nblock_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "bc11c716",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['oCountry', 'tCountry', 'Significant', 'edge_preference', 'FieldDistance', 'oPop', 'oGDP', 'oGDP_PCAP', 'oGNI', 'oGNI_PCAP', 'oRND_per', 'tPop', 'tGDP', 'tGDP_PCAP', 'tGNI', 'tGNI_PCAP', 'tRND_per', 'PhysicalDistance', 'tradeflow_baci', 'comlang_off', 'CollabStrength', 'DirectedCollabStrength', 'CosineNormStrength', 'oTopFrac', 'tTopFrac', 'oDemoc', 'tDemoc', 'DemocDis', 'oContinent', 'tContinent', 'SameContinent', 'Agreements']\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.322663\n",
      "         Iterations 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.281216\n",
      "         Iterations 9\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.266267\n",
      "         Iterations 9\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.249058\n",
      "         Iterations 10\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.232618\n",
      "         Iterations 10\n"
     ]
    }
   ],
   "source": [
    "#network_panel=pd.read_csv('Regression/network_panel_edge_preference_polity_2017.csv')\n",
    "#network_panel=pd.read_csv('Regression/network_panel_edge_preference_polity_continent_2017.csv')\n",
    "network_panel=pd.read_csv('Regression/network_panel_edge_preference_collab_top_polity_continent_agreement_2017.csv')\n",
    "print(list(network_panel.columns))\n",
    "network_panel['edge_preference'].replace(-1, 2, inplace=True)\n",
    "network_panel['DemocDis']=network_panel['DemocDis'].map(lambda x:abs(x)) #  ensure the democracy distance is \n",
    "\n",
    "# 'SameContinent', 'oGDP_PCAP','tGDP_PCAP'\n",
    "Model_features = ['oGDP_PCAP','tGDP_PCAP','oPop','tPop','PhysicalDistance', 'SameContinent', 'FieldDistance',\n",
    "                  'tradeflow_baci','oTopFrac','tTopFrac','oDemoc','tDemoc','DemocDis','CollabStrength']\n",
    "\n",
    "dummies=['comlang_off','Agreements']\n",
    "\n",
    "dependent_var = 'edge_preference'\n",
    "\n",
    "df = network_panel[[dependent_var]+Model_features + dummies]\n",
    "\n",
    "\n",
    "#v1=['oGNI_PCAP','tGNI_PCAP', 'oPop','tPop']\n",
    "v1=['oGDP_PCAP','tGDP_PCAP', 'oPop','tPop']\n",
    "v2=['PhysicalDistance','SameContinent','FieldDistance','tradeflow_baci']\n",
    "v3=['comlang_off', 'oDemoc','tDemoc','DemocDis','Agreements']\n",
    "v4=['CollabStrength'] #CollabStrength\n",
    "v5=['oTopFrac','tTopFrac']\n",
    "\n",
    "exog_var_sets = [v1,v2,v3,v4,v5]\n",
    "dep_var = \"edge_preference\"\n",
    "\n",
    "Nmodels = len(exog_var_sets)\n",
    "\n",
    "fit_models = []\n",
    "for icol in range(Nmodels):\n",
    "    exog_vars = list(chain.from_iterable(exog_var_sets[:(icol+1)]))\n",
    "    X = df[exog_vars + [dep_var]].dropna().reset_index(drop=True)\n",
    "\n",
    "    Y = X[dep_var]\n",
    "    X = sm.add_constant(X[exog_vars])\n",
    "\n",
    "    for c in exog_vars:\n",
    "        if X[c].nunique() > 2:\n",
    "            X[c] = (X[c] - X[c].mean())/X[c].std()\n",
    "\n",
    "    model = sm.MNLogit(Y, X).fit()\n",
    "    fit_models.append(model)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "98d0c2bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def summary_table(model,ycat,ycat_name):\n",
    "    \n",
    "    dfpo=model.params.reset_index().rename(columns={'index':'item'})\n",
    "    dfc1=dfpo[['item',ycat-1]].rename(columns={ycat-1:'risk'})\n",
    "    \n",
    "    dfpv=model.pvalues.reset_index().rename(columns={'index':'item'})\n",
    "    dfpv=dfpv[['item',ycat-1]].rename(columns={ycat-1:'pvalues'})\n",
    "\n",
    "    df1=dfc1.merge(dfpv,how='left',on='item')\n",
    "    \n",
    "    dfcon1=model.conf_int().loc[str(ycat)].reset_index().rename(columns={'index':'item','lower':0,'upper':1})\n",
    "    df2=df1.merge(dfcon1,how='left',on='item')\n",
    "\n",
    "    df2['strf']=ycat_name\n",
    "\n",
    "    return df2\n",
    "\n",
    "dfc1=summary_table(fit_models[-1],1,'Positive recognition')\n",
    "dfc2=summary_table(fit_models[-1],2,'Negative recognition')\n",
    "\n",
    "dfs=pd.concat([dfc1,dfc2])\n",
    "\n",
    "namedict = {'const':'Intercept', \n",
    "            'oTopCumuFrac':'Source top journal fraction', 'tTopCumuFrac':'Target top journal fraction',\n",
    "            'oPop':'Source population','tPop':'Target population',\n",
    "             'oGDP_PCAP':'Source GDP per capita', 'tGDP_PCAP':'Target GDP per capita',\n",
    "            'oTopFrac':'Source top journal fraction', 'tTopFrac':'Target top journal fraction',\n",
    "            'PhysicalDistance': 'Physical distance', 'tradeflow_baci':'Trade volume', \n",
    "            'FieldDistance':'Topic distance', 'SameContinent':'Same continent',\n",
    "            'CollabStrength':'Collaboration strength',\n",
    "            'CosineNormStrength':'Normed collaboration strength',\n",
    "            'comlang_off':'Same official language', \n",
    "            'DemocDis': 'Democracy distance','oDemoc':'Source democracy', 'tDemoc':'Target democracy',\n",
    "           'oGNI_PCAP':'Source GNI per capita', 'tGNI_PCAP':'Target GNI per capita',\n",
    "           'GNI_PCAP':'GNI per capita', 'GNI':'GNI', 'GDP_PCAP':'GDP per capita',\n",
    "           'Pop':'Population', 'NumPub':'Number of publications','Agreements':'Bilateral research agreement',\n",
    "           'RND_per':'Percent of GDP for R&D', \"FractionNationalAuthors\":\"Fraction national authors\"}\n",
    "\n",
    "dfs.replace(namedict,inplace=True)\n",
    "\n",
    "dfs['category']=dfs['item']\n",
    "\n",
    "df1 = dfs[dfs[\"strf\"] == \"Positive recognition\"].reset_index(drop=True)\n",
    "df2 = dfs[dfs[\"strf\"] == \"Negative recognition\"].reset_index(drop=True)\n",
    "\n",
    "df1['xerror']=df1[1]-df1['risk']\n",
    "df2['xerror']=df2[1]-df2['risk']\n",
    "\n",
    "df1=df1[df1['item']!='Intercept']\n",
    "df2=df2[df2['item']!='Intercept']\n",
    "\n",
    "#dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ecff68ed-b8f6-46e3-951a-5dba0901f7be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item</th>\n",
       "      <th>risk</th>\n",
       "      <th>pvalues</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>strf</th>\n",
       "      <th>category</th>\n",
       "      <th>xerror</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Source GNI per capita</td>\n",
       "      <td>-0.092477</td>\n",
       "      <td>6.687006e-01</td>\n",
       "      <td>-0.516027</td>\n",
       "      <td>0.331074</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Source GNI per capita</td>\n",
       "      <td>0.423550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Target GNI per capita</td>\n",
       "      <td>-0.569696</td>\n",
       "      <td>3.837964e-03</td>\n",
       "      <td>-0.955899</td>\n",
       "      <td>-0.183493</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Target GNI per capita</td>\n",
       "      <td>0.386203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Source population</td>\n",
       "      <td>0.829810</td>\n",
       "      <td>1.235295e-05</td>\n",
       "      <td>0.457744</td>\n",
       "      <td>1.201876</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Source population</td>\n",
       "      <td>0.372066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Target population</td>\n",
       "      <td>0.151840</td>\n",
       "      <td>4.359508e-01</td>\n",
       "      <td>-0.230165</td>\n",
       "      <td>0.533844</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Target population</td>\n",
       "      <td>0.382005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Physical distance</td>\n",
       "      <td>-0.508793</td>\n",
       "      <td>4.464942e-04</td>\n",
       "      <td>-0.792823</td>\n",
       "      <td>-0.224764</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Physical distance</td>\n",
       "      <td>0.284030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Same continent</td>\n",
       "      <td>0.565007</td>\n",
       "      <td>2.512394e-02</td>\n",
       "      <td>0.070523</td>\n",
       "      <td>1.059491</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Same continent</td>\n",
       "      <td>0.494484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Topic distance</td>\n",
       "      <td>-1.492858</td>\n",
       "      <td>2.240516e-09</td>\n",
       "      <td>-1.982201</td>\n",
       "      <td>-1.003514</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Topic distance</td>\n",
       "      <td>0.489343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Trade volume</td>\n",
       "      <td>-0.450450</td>\n",
       "      <td>2.855107e-02</td>\n",
       "      <td>-0.853653</td>\n",
       "      <td>-0.047247</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Trade volume</td>\n",
       "      <td>0.403203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Same official language</td>\n",
       "      <td>1.215920</td>\n",
       "      <td>3.946199e-07</td>\n",
       "      <td>0.746011</td>\n",
       "      <td>1.685828</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Same official language</td>\n",
       "      <td>0.469909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Source democracy</td>\n",
       "      <td>-0.429975</td>\n",
       "      <td>9.126712e-04</td>\n",
       "      <td>-0.684106</td>\n",
       "      <td>-0.175844</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Source democracy</td>\n",
       "      <td>0.254131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Target democracy</td>\n",
       "      <td>-0.691070</td>\n",
       "      <td>7.962755e-08</td>\n",
       "      <td>-0.943395</td>\n",
       "      <td>-0.438745</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Target democracy</td>\n",
       "      <td>0.252325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Democracy distance</td>\n",
       "      <td>-0.450734</td>\n",
       "      <td>2.234773e-03</td>\n",
       "      <td>-0.739706</td>\n",
       "      <td>-0.161761</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Democracy distance</td>\n",
       "      <td>0.288972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Collaboration strength</td>\n",
       "      <td>3.079645</td>\n",
       "      <td>3.109679e-24</td>\n",
       "      <td>2.485329</td>\n",
       "      <td>3.673962</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Collaboration strength</td>\n",
       "      <td>0.594316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Source top journal fraction</td>\n",
       "      <td>0.045370</td>\n",
       "      <td>7.611448e-01</td>\n",
       "      <td>-0.247162</td>\n",
       "      <td>0.337902</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Source top journal fraction</td>\n",
       "      <td>0.292532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Target top journal fraction</td>\n",
       "      <td>1.145963</td>\n",
       "      <td>2.872893e-14</td>\n",
       "      <td>0.850583</td>\n",
       "      <td>1.441343</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Target top journal fraction</td>\n",
       "      <td>0.295380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Bilateral agreements</td>\n",
       "      <td>-0.816407</td>\n",
       "      <td>1.401192e-04</td>\n",
       "      <td>-1.236614</td>\n",
       "      <td>-0.396201</td>\n",
       "      <td>Positive recognition</td>\n",
       "      <td>Bilateral agreements</td>\n",
       "      <td>0.420207</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           item      risk       pvalues         0         1  \\\n",
       "1         Source GNI per capita -0.092477  6.687006e-01 -0.516027  0.331074   \n",
       "2         Target GNI per capita -0.569696  3.837964e-03 -0.955899 -0.183493   \n",
       "3             Source population  0.829810  1.235295e-05  0.457744  1.201876   \n",
       "4             Target population  0.151840  4.359508e-01 -0.230165  0.533844   \n",
       "5             Physical distance -0.508793  4.464942e-04 -0.792823 -0.224764   \n",
       "6                Same continent  0.565007  2.512394e-02  0.070523  1.059491   \n",
       "7                Topic distance -1.492858  2.240516e-09 -1.982201 -1.003514   \n",
       "8                  Trade volume -0.450450  2.855107e-02 -0.853653 -0.047247   \n",
       "9        Same official language  1.215920  3.946199e-07  0.746011  1.685828   \n",
       "10             Source democracy -0.429975  9.126712e-04 -0.684106 -0.175844   \n",
       "11             Target democracy -0.691070  7.962755e-08 -0.943395 -0.438745   \n",
       "12           Democracy distance -0.450734  2.234773e-03 -0.739706 -0.161761   \n",
       "13       Collaboration strength  3.079645  3.109679e-24  2.485329  3.673962   \n",
       "14  Source top journal fraction  0.045370  7.611448e-01 -0.247162  0.337902   \n",
       "15  Target top journal fraction  1.145963  2.872893e-14  0.850583  1.441343   \n",
       "16         Bilateral agreements -0.816407  1.401192e-04 -1.236614 -0.396201   \n",
       "\n",
       "                    strf                     category    xerror  \n",
       "1   Positive recognition        Source GNI per capita  0.423550  \n",
       "2   Positive recognition        Target GNI per capita  0.386203  \n",
       "3   Positive recognition            Source population  0.372066  \n",
       "4   Positive recognition            Target population  0.382005  \n",
       "5   Positive recognition            Physical distance  0.284030  \n",
       "6   Positive recognition               Same continent  0.494484  \n",
       "7   Positive recognition               Topic distance  0.489343  \n",
       "8   Positive recognition                 Trade volume  0.403203  \n",
       "9   Positive recognition       Same official language  0.469909  \n",
       "10  Positive recognition             Source democracy  0.254131  \n",
       "11  Positive recognition             Target democracy  0.252325  \n",
       "12  Positive recognition           Democracy distance  0.288972  \n",
       "13  Positive recognition       Collaboration strength  0.594316  \n",
       "14  Positive recognition  Source top journal fraction  0.292532  \n",
       "15  Positive recognition  Target top journal fraction  0.295380  \n",
       "16  Positive recognition         Bilateral agreements  0.420207  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "1c6c6d4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source GDP per capita (-5.111989857485999, 3.1878294656067625e-07)\n",
      "Target GDP per capita (-6.8053692331690465, 1.0079039442660139e-11)\n",
      "Source population (-6.14584782476865, 7.953751154438744e-10)\n",
      "Target population (-8.527211828903479, 1.4991719615882735e-17)\n",
      "Physical distance (-0.8668155067897941, 0.38604310430218136)\n",
      "Same continent (5.880785356364366, 4.083243944465331e-09)\n",
      "Topic distance (-5.656725877155015, 1.5428788648705385e-08)\n",
      "Trade volume (-0.6510036661135686, 0.5150441203226309)\n",
      "Same official language (5.784964801830892, 7.252751471734106e-09)\n",
      "Source democracy (-2.3998236972873057, 0.016402969952863024)\n",
      "Target democracy (-2.5031905534685865, 0.012307925775734654)\n",
      "Democracy distance (-4.554026978359761, 5.262860217362792e-06)\n",
      "Bilateral research agreement (-4.504142152164928, 6.66415889442e-06)\n",
      "Collaboration strength (8.443900450036116, 3.069206044102059e-17)\n",
      "Source top journal fraction (-5.144907044861672, 2.6765330827431096e-07)\n",
      "Target top journal fraction (8.21067696684086, 2.19944878808394e-16)\n"
     ]
    }
   ],
   "source": [
    "def wald_mlogit_diff(model, param1, param2=None):\n",
    "    covar = model.cov_params()\n",
    "    flipvalue = {'1':'2', '2':'1'}\n",
    "    if param2 is None:\n",
    "        param2 = \":\".join([flipvalue.get(p, p) for p in param1.split(':')])\n",
    "\n",
    "    pidx1 = tuple(param1.split(':'))[::-1]\n",
    "    pidx2 = tuple(param2.split(':'))[::-1]\n",
    "\n",
    "    diffse = np.sqrt( covar[pidx1][pidx1] + covar[pidx2][pidx2] - 2*covar[pidx1][pidx2] )\n",
    "\n",
    "    coeffdiff = model.params[int(pidx1[0])-1][pidx1[1]] - model.params[int(pidx2[0])-1][pidx2[1]]\n",
    "    zscore = coeffdiff/diffse\n",
    "    pvalue = spstats.norm.sf(np.abs(zscore))*2\n",
    "    \n",
    "    return zscore, pvalue\n",
    "\n",
    "for it in dfc1['item'].values:\n",
    "    if it != 'const':\n",
    "        param1 = it+':1'\n",
    "        print(namedict[it], wald_mlogit_diff(fit_models[-1], param1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "757b2de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "colours=['#8D95A0','#2171b5','#DA6437']\n",
    "colours=['#8D95A0','#2171b5',sns.color_palette()[3]]\n",
    "\n",
    "def logistic_coefficient_plot(df1, df2, ax=None):\n",
    "    \n",
    "    if ax is None:\n",
    "        fig, ax = plt.subplots(figsize=(5,12))\n",
    "    \n",
    "    \n",
    "    df1s=df1[df1['pvalues']<=0.05]\n",
    "    df1ns=df1[df1['pvalues']>0.05]\n",
    "\n",
    "    df2s=df2[df2['pvalues']<=0.05]\n",
    "    df2ns=df2[df2['pvalues']>0.05]\n",
    "\n",
    "    x1s = df1s['risk']\n",
    "    y1s = np.array(df1s.index.to_list())-0.2\n",
    "    xerror1s = df1s['xerror']\n",
    "\n",
    "    x1ns = df1ns['risk']\n",
    "    y1ns = np.array(df1ns.index.to_list())-0.2\n",
    "    xerror1ns = df1ns['xerror']\n",
    "\n",
    "    x2s = df2s['risk']\n",
    "    y2s = np.array(df2s.index.to_list())+0.2\n",
    "    xerror2s = df2s['xerror']\n",
    "\n",
    "    x2ns = df2ns['risk']\n",
    "    y2ns = np.array(df2ns.index.to_list())+0.2\n",
    "    xerror2ns = df2ns['xerror']\n",
    "    \n",
    "    y=df1.index.to_list()\n",
    "    \n",
    "    ax.errorbar(x1s, y1s, xerr=xerror1s, fmt='o',color=colours[1],label=\"Positive recognition\",ms=4)\n",
    "    ax.errorbar(x2s, y2s, xerr=xerror2s, fmt='o',color=colours[2],label=\"Negative recognition\",ms=4)\n",
    "    \n",
    "    ax.errorbar(x1ns, y1ns, xerr=xerror1ns, fmt='o',color=colours[1],mfc='w',mew=1,label=\"Positive recognition(not sig)\",ms=4)\n",
    "    ax.errorbar(x2ns, y2ns, xerr=xerror2ns, fmt='o',color=colours[2],mfc='w',mew=1,label=\"Negative recognition(not sig)\",ms=4)\n",
    "\n",
    "    #ax.errorbar(x1ns, y1ns, xerr=xerror1ns, fmt='o',color=colours[1],label=\"Positive recognition(not sig)\",ms=4, alpha=0.5)\n",
    "    #ax.errorbar(x2ns, y2ns, xerr=xerror2ns, fmt='o',color=colours[2],label=\"Negative recognition(not sig)\",ms=4, alpha=0.5)\n",
    "\n",
    "    ax.invert_yaxis()\n",
    "    \n",
    "    ax.axvline(x=0, color=colours[0],  linestyle='--',ymin=0,ymax=0.98)\n",
    "\n",
    "    sns.despine(top=True, right=True, left=True, bottom=False)\n",
    "\n",
    "    ylabels=df1['item']\n",
    "    ax.set_yticks(y)\n",
    "    ax.set_yticklabels(ylabels)\n",
    "    ax.set_xticks([-3,-2,-1,0,1,2,3,4])\n",
    "    ax.tick_params(axis=\"both\", which=\"both\", bottom=True, top=False,    \n",
    "                    labelbottom=True, left=False, right=False, labelleft=True,labelsize=9) \n",
    "\n",
    "    ax.set_ylim([len(Model_features)+2.5,0])\n",
    "    ax.set_xlabel(\"Regression coefficient\",fontsize=10)\n",
    "\n",
    "    hlines=y\n",
    "    counter=0\n",
    "    for ix in hlines: \n",
    "        if counter % 2 == 0:\n",
    "            ax.axhspan(ix - 0.5, ix + 0.5, color=colours[0],alpha=0.3, zorder=0,lw=0)\n",
    "        counter += 1\n",
    "\n",
    "\n",
    "    plt.legend(bbox_to_anchor=(-0.3, 0.55, 0.5, 0.5),\n",
    "        fontsize=10,\n",
    "               loc=\"upper left\",\n",
    "               ncol=2,\n",
    "               markerscale=1,\n",
    "               frameon=False,\n",
    "               handletextpad=.1,\n",
    "               columnspacing=.2)\n",
    "    \n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "6925cbd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 183584 characters]\n",
      "text/plain": [
       "<Figure size 1000x600 with 5 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "countries2highlight = sorted(['US', 'CH', 'GB', 'PH', 'SA', 'SG', 'CN', 'IN'])\n",
    "\n",
    "lblsize = 9\n",
    "\n",
    "layout = [\n",
    "    [\"A\", \"B\", \"C\"],\n",
    "    [\"A\", \"D\", \"E\"]\n",
    "]\n",
    "\n",
    "fig, axd = plt.subplot_mosaic(layout, figsize=(10,6))\n",
    "\n",
    "logistic_coefficient_plot(df1, df2, ax=axd[\"A\"])\n",
    "\n",
    "label = \"A\"\n",
    "trans = mtransforms.ScaledTranslation(-2, 7/72, fig.dpi_scale_trans)\n",
    "axd[label].text(0.0, 1.0, label, transform=axd[label].transAxes + trans,\n",
    "                   fontsize='x-large', va='center')\n",
    "\n",
    "comm2color = {0:\"#257BA0\", 1:\"#6AB8AB\", 2:\"#D65451\", 3:\"#FFE000\", 4:\"#FF9400\"}\n",
    "\n",
    "for c in countryprod['Country'].unique():\n",
    "    countrysub = countryprod[countryprod['Country'] == c].sort_values('Year')\n",
    "    \n",
    "    if c in countries2highlight:\n",
    "        color = sns.color_palette(\"rocket\", n_colors=len(countries2highlight) + 2)[countries2highlight.index(c)]\n",
    "        axd[\"B\"].plot(countrysub['Year'].values, countrysub['PositiveInDegree'].values, color=color, lw=2, zorder=10)\n",
    "        axd[\"C\"].plot(countrysub['Year'].values, countrysub['NegativeInDegree'].values, color=color, lw=2, zorder=10)\n",
    "        \n",
    "        if c in ['CN', 'SA', 'SG']:\n",
    "            axd[\"B\"].text(s=c, x=countrysub['Year'].values[-1], y=countrysub['PositiveInDegree'].values[-1],\n",
    "                    ha='left', va='top', color=color, fontsize=8)\n",
    "        else:\n",
    "            axd[\"B\"].text(s=c, x=countrysub['Year'].values[-1], y=countrysub['PositiveInDegree'].values[-1],\n",
    "                    ha='left', va='bottom', color=color, fontsize=8)\n",
    "        \n",
    "        if c in ['IN', 'PH', 'CH', 'SG']:\n",
    "            axd[\"C\"].text(s=c, x=countrysub['Year'].values[-1], y=countrysub['NegativeInDegree'].values[-1],\n",
    "                    ha='left', va='top', color=color, fontsize=8)\n",
    "        elif c in ['GB']:\n",
    "            axd[\"C\"].text(s=c, x=countrysub['Year'].values[-1], y=countrysub['NegativeInDegree'].values[-1],\n",
    "                    ha='left', va='center', color=color, fontsize=8)\n",
    "        else:\n",
    "            axd[\"C\"].text(s=c, x=countrysub['Year'].values[-1], y=countrysub['NegativeInDegree'].values[-1],\n",
    "                    ha='left', va='bottom', color=color, fontsize=8)\n",
    "    else:    \n",
    "        axd[\"B\"].plot(countrysub['Year'].values, countrysub['PositiveInDegree'].values, color='0.7', lw=0.1, zorder=1)\n",
    "        axd[\"C\"].plot(countrysub['Year'].values, countrysub['NegativeInDegree'].values, color='0.7', lw=0.1, zorder=1)\n",
    "\n",
    "axd[\"B\"].set_ylabel('Positive in-degree', fontsize=lblsize)\n",
    "axd[\"C\"].set_ylabel('Negative in-degree', fontsize=lblsize)\n",
    "axd[\"D\"].set_ylabel('Normalized entropy', fontsize=lblsize)\n",
    "axd[\"E\"].set_ylabel('Negative connection probability', fontsize=lblsize)\n",
    "\n",
    "#axd[\"E\"].axis('off')\n",
    "\n",
    "axd['D'].plot(np.arange(1990, 2018), ygini, color='0.8', lw=2)\n",
    "\n",
    "for i,(ic,jc) in enumerate([(1,2), (2,1), (2,3),  (2,0),(0,3), (0,2), (3,2)]):\n",
    "    color=sns.color_palette(\"rocket\", n_colors=len(countries2highlight) + 2)[i]\n",
    "    axd[\"E\"].plot(np.arange(1990, 2018), nblock_matrix[:,ic,jc], lw=2, color=color)\n",
    "    \n",
    "    if (ic,jc) in [(3,0),  (2,3)]:\n",
    "        va = 'top'\n",
    "    elif (ic,jc) in [(3,2), (3,0)]:\n",
    "        va = 'center'\n",
    "    else:\n",
    "        va = 'bottom'\n",
    "    axd[\"E\"].text(s=str(ic+1)+\"->\"+str(jc+1), x=2017, y=nblock_matrix[-1,ic,jc], ha='left', va=va, \n",
    "                  color=color, fontsize=8)\n",
    "\n",
    "#axd[\"E\"].set_yticks([0, 0.25, 0.5,0.75])\n",
    "axd[\"E\"].set_yticks([0, 0.25, 0.5,0.75])\n",
    "\n",
    "\n",
    "for label in ['B', \"C\", \"D\", \"E\"]:\n",
    "    axd[label].set_xlabel('Year', fontsize=lblsize)\n",
    "    # label physical distance in and down:\n",
    "    trans = mtransforms.ScaledTranslation(-0.65, 7/72, fig.dpi_scale_trans)\n",
    "    axd[label].text(0.0, 1.0, label, transform=axd[label].transAxes + trans,\n",
    "                   fontsize='x-large', va='bottom')#, xx-small, x-small, small, medium, large, x-large, xx-large, larger, smaller\n",
    "plt.tight_layout()\n",
    "\n",
    "plt.savefig('Figures/Fig2_networkfeatures.pdf')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2967b56c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 33176 characters]\n",
      "text/plain": [
       "<Figure size 600x600 with 25 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.gridspec as gridspec\n",
    "\n",
    "fig = plt.figure(figsize = (6,6))\n",
    "gs = gridspec.GridSpec(nc,nc)\n",
    "gs.update(wspace=0.1, hspace=0.1) # set the spacing between axes. \n",
    "\n",
    "fontProperties = {'family':'sans-serif','sans-serif':['Helvetica'],\n",
    "    'weight' : 'normal', 'size' : 12}\n",
    "\n",
    "for ic in fullcnames:\n",
    "    for jc in fullcnames:\n",
    "        \n",
    "        ax = plt.subplot(gs[ic * nc + jc])\n",
    "        \n",
    "        #pyears = pblock_matrix[:,ic,jc] > 0\n",
    "        #nyears = nblock_matrix[:,ic,jc] > 0\n",
    "        \n",
    "        #ax.plot(np.arange(1990, 2018)[pyears], pblock_matrix[:,ic,jc][pyears], lw=2)\n",
    "        #ax.plot(np.arange(1990, 2018)[nyears], nblock_matrix[:,ic,jc][nyears], lw=2, color=sns.color_palette()[3])\n",
    "        \n",
    "        ax.plot(np.arange(1990, 2018), pblock_matrix[:,ic,jc], lw=2)\n",
    "        ax.plot(np.arange(1990, 2018), nblock_matrix[:,ic,jc], lw=2, color=sns.color_palette()[3])\n",
    "        \n",
    "        ax.set_ylim([0, 0.8])\n",
    "        ax.set_yticks([0,0.75])\n",
    "        \n",
    "        ax.tick_params(axis='y', which='major', labelsize=10)\n",
    "        for tick in ax.get_yticklabels():\n",
    "            tick.set_fontname(\"Helvetica\") #\"Comic Sans MS\")\n",
    "        \n",
    "        for tick in ax.get_xticklabels():\n",
    "            tick.set_fontname(\"Helvetica\")\n",
    "            tick.set_rotation(45)\n",
    "\n",
    "        #if ~(ic == 0 and jc ==2 or ic==2 and jc ==0):\n",
    "        ax.spines['top'].set_visible(False)\n",
    "        ax.spines['right'].set_visible(False)\n",
    "        ax.spines['bottom'].set_visible(False)\n",
    "        ax.spines['left'].set_visible(False)\n",
    "        \n",
    "        ax.set_facecolor('0.9')\n",
    "        \n",
    "        if jc > -1:\n",
    "            ax.set_yticks([])\n",
    "        \n",
    "        if ic <6:\n",
    "            ax.set_xticks([])\n",
    "#plt.tight_layout()  \n",
    "\n",
    "plt.savefig(\"Figures/CommunityInteraction_12_23.pdf\", bbox_inches='tight')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f320a1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def line_stick_auc(ax, auc_data, countries_to_highlight):\n",
    "    ax.axhline(y=0.6, ls='-', color='0.8', lw=0.5, zorder=1)\n",
    "    #ax.axhline(y=0.55, ls='-', color='0.8', lw=0.5, zorder=1)\n",
    "    #ax.axhline(y=0.65, ls='-', color='0.8', lw=0.5, zorder=1)\n",
    "    ax.axhline(y=0.7, ls='-', color='0.8', lw=0.5, zorder=1)\n",
    "    ax.axhline(y=0.8, ls='-', color='0.8', lw=0.5, zorder=1)\n",
    "    \n",
    "    auc_data = auc_data.sort_values(by='AUC')\n",
    "    \n",
    "    for i, (auc, cname) in enumerate(auc_data[['AUC', 'Country']].values):\n",
    "        if cname in countries_to_highlight:\n",
    "            ax.plot([i,i], [0.5,auc], color = 'k')\n",
    "            ax.scatter(i,auc, color='k')\n",
    "        else:\n",
    "            ax.plot([i,i], [0.5,auc], color = '0.6')\n",
    "            ax.scatter(i,auc, color='0.6')\n",
    "    \n",
    "    ax.set_xlim([-0.5, auc_data['Country'].nunique()])\n",
    "    \n",
    "    ax.set_xticks(np.arange(auc_data['Country'].nunique()))\n",
    "    ax.set_xticklabels(auc_data['Country'].values, \n",
    "                       rotation=45, ha='center', color='0.6', fontsize=8)\n",
    "    for i in np.where(pyscisci.isin_sorted(auc_data['Country'].values, np.sort(countries_to_highlight)))[0]:\n",
    "        ax.get_xticklabels()[i].set_color(\"k\")\n",
    "\n",
    "    ax.set_ylim([0.5, 0.82])\n",
    "    \n",
    "    ax.set_yticks([0.5, 0.6, 0.7, 0.8])\n",
    "    ax.tick_params(axis='y', size=0)\n",
    "    #ax.set_ylabel('Over-representation')\n",
    "    for s in ['top', 'right', 'left']:\n",
    "        ax.spines[s].set_visible(False)\n",
    "    \n",
    "    return ax\n",
    "\n",
    "def mycoefplot(ax, sumdf, fontsize=10, ms=50):\n",
    "    exog_vars = sumdf.index\n",
    "    i=len(exog_vars) -1\n",
    "    for f in exog_vars:\n",
    "        if f in exog_vars:\n",
    "            coeff, stderr, pvalue, lower, upper = sumdf.loc[f][['Parameter', 'Std. Err.','P-value', 'Lower CI', 'Upper CI']]\n",
    "            \n",
    "            #color = sns.color_palette()[1]\n",
    "            color = sns.color_palette(\"rocket\", n_colors = 4)[1]\n",
    "            if pvalue >= 0.05:\n",
    "                fc = 'w'\n",
    "            else:\n",
    "                fc = color\n",
    "\n",
    "            ax.plot([lower, upper], [i]*2, color=color, lw=2, zorder=5)\n",
    "            ax.scatter(coeff, i, s=ms, color=fc, edgecolors = color, zorder=10)\n",
    "            \n",
    "            if False:\n",
    "                ax.text(1.9, i, \"(S.E.={},P={})\".format(np.round(stderr, 2), np.round(pvalue, 2)), fontsize=14, va='center')\n",
    "            i-=1\n",
    "\n",
    "    ax.grid(axis='x')\n",
    "    ax.axvline(x=0, lw=2, color='k')\n",
    "    ax.set_xlim([-2.5, 1.5])\n",
    "    ax.set_xticks([-2,-1,0,1])\n",
    "    ax.set_xlabel('Model coefficient', fontsize=fontsize)\n",
    "    ax.set_ylim([-1, len(exog_vars)])\n",
    "\n",
    "    ax.set_yticks(list(range(len(exog_vars))))\n",
    "\n",
    "    if True:\n",
    "        ax.set_yticklabels([namedict.get(fname, fname) for fname in exog_vars[::-1]], \n",
    "                             fontdict = {'rotation':0, 'horizontalalignment':'right',\n",
    "                                        'fontsize':fontsize})    \n",
    "\n",
    "    ax.tick_params(axis='x', which='major', labelsize=fontsize)\n",
    "    \n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "45820a3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "self_auc = country_auc_df[country_auc_df['CitingCountry'] == country_auc_df['CitedCountry']].sort_values(by=['CitedYear', 'CitedCountry']).reset_index(drop=True)\n",
    "self_auc = self_auc[self_auc['Significant']]\n",
    "self_auc.rename(columns={'CitedYear':'Year', 'CitedCountry':'Country'}, inplace=True)\n",
    "\n",
    "countryprod = pd.read_csv('CountryData/CountryTopProductivity.csv')\n",
    "countryprod['NumPub'] = np.log10(countryprod['NumPub'])\n",
    "\n",
    "self_auc = self_auc.merge(countryprod, how='left', on=['Country', 'Year'])\n",
    "\n",
    "oasize = pd.read_csv('CountryData/globaltopjournal.csv')\n",
    "self_auc = self_auc.merge(oasize, how='left', on='Year')\n",
    "#patents = pd.read_csv('/Users/hgt6rn/Downloads/resulttable.csv', sep=';')\n",
    "#patents = patents.rename(columns={'appln_filing_year':'Year', 'person_ctry_code':'Country'})\n",
    "\n",
    "#self_auc = self_auc.merge(patents, how='left', on=['Year', 'Country'])\n",
    "\n",
    "self_auc['NormedTop'] = self_auc['FracTop'].values/self_auc['OAFracTop'].values\n",
    "\n",
    "national_authorship_rate = pd.read_csv( \"CountryData/oa_national_authorship_rate.csv.gz\")\n",
    "self_auc = self_auc.merge(national_authorship_rate, how='left', on=['Country', 'Year'])\n",
    "\n",
    "pubtopics1_div = pd.read_csv(\"CountryData/oa_country_topic1_diversity.csv.gz\")\n",
    "self_auc = self_auc.merge(pubtopics1_div, how='left', on=['Country', 'Year'])\n",
    "self_auc.rename(columns={'TopicDiversity':'TopicDiversity1'}, inplace=True)\n",
    "self_auc['TopicDiversity1'] = 1 - self_auc['TopicDiversity1']\n",
    "\n",
    "pubtopics2_div = pd.read_csv(\"CountryData/oa_country_topic2_diversity.csv.gz\")\n",
    "self_auc = self_auc.merge(pubtopics2_div, how='left', on=['Country', 'Year'])\n",
    "self_auc.rename(columns={'TopicDiversity':'TopicDiversity2'}, inplace=True)\n",
    "self_auc['TopicDiversity2'] = 1 - self_auc['TopicDiversity2']\n",
    "\n",
    "wbgdp = pd.read_csv('CountryData/world-bank-data.csv')\n",
    "wbgdp['PAT_total'] = wbgdp['PAT_nres'] + wbgdp['PAT_res']\n",
    "for c in ['GDP', 'GNI', 'GDP_PCAP', 'GNI_PCAP', 'PAT_nres', 'PAT_res', 'PAT_total', 'RND_per', 'Pop']:   #'GNI_PCAP', 'GDP_PCAP', 'RND_per'\n",
    "    wbgdp[c] = np.log10(wbgdp[c].astype(float))\n",
    "\n",
    "self_auc = self_auc.merge(wbgdp, how='left', on=['Country', 'Year'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "86c060cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          PanelOLS Estimation Summary                           \n",
      "================================================================================\n",
      "Dep. Variable:              NormedTop   R-squared:                        0.3916\n",
      "Estimator:                   PanelOLS   R-squared (Between):             -0.0015\n",
      "No. Observations:                 805   R-squared (Within):               0.3782\n",
      "Date:                Thu, Jan 25 2024   R-squared (Overall):              0.1185\n",
      "Time:                        12:50:56   Log-likelihood                    104.92\n",
      "Cov. Estimator:             Clustered                                           \n",
      "                                        F-statistic:                      76.171\n",
      "Entities:                          68   P-value                           0.0000\n",
      "Avg Obs:                       11.838   Distribution:                   F(6,710)\n",
      "Min Obs:                       1.0000                                           \n",
      "Max Obs:                       22.000   F-statistic (robust):             11.138\n",
      "                                        P-value                           0.0000\n",
      "Time periods:                      22   Distribution:                   F(6,710)\n",
      "Avg Obs:                       36.591                                           \n",
      "Min Obs:                       18.000                                           \n",
      "Max Obs:                       53.000                                           \n",
      "                                                                                \n",
      "                                    Parameter Estimates                                    \n",
      "===========================================================================================\n",
      "                         Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
      "-------------------------------------------------------------------------------------------\n",
      "Pop                        -0.5326     0.6263    -0.8505     0.3954     -1.7621      0.6969\n",
      "GDP_PCAP                    0.4367     0.3497     1.2488     0.2121     -0.2498      1.1232\n",
      "NumPub                     -0.0808     0.2149    -0.3761     0.7069     -0.5028      0.3411\n",
      "RND_per                     0.0086     0.0673     0.1282     0.8980     -0.1236      0.1409\n",
      "FractionNationalAuthors    -0.5903     0.0892    -6.6166     0.0000     -0.7654     -0.4151\n",
      "AUC                         0.1645     0.0748     2.1978     0.0283      0.0176      0.3114\n",
      "===========================================================================================\n",
      "\n",
      "F-test for Poolability: 39.228\n",
      "P-value: 0.0000\n",
      "Distribution: F(88,710)\n",
      "\n",
      "Included effects: Entity, Time\n",
      "\\begin{table}[!htb] \\centering \n",
      "    \\small\n",
      "  \\caption{\\textbf{National self-preference.}\n",
      "  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \n",
      "  \\label{table:} \n",
      "  \\begin{tabular}{lllll}\\\\[-1.8ex]\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\multicolumn{5}{c}{\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\[0.8ex]\\hline \n",
      " & \\multicolumn{4}{c}{Model}  \\\\ \\cline{2-5} \\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\ \\hline \\\\[-1.8ex] \n",
      "Population & $-1.41$ & $0.55$ & $-0.38$ & $-0.53$\\\\ \n",
      " & $(-3.13,0.3)$ & $(-1.36,2.46)$ & $(-1.65,0.88)$ & $(-1.76,0.7)$\\\\ \n",
      " & S.E. $0.88$ p=$0.1068$ & S.E. $0.97$ p=$0.5726$ & S.E. $0.64$ p=$0.5536$ & S.E. $0.63$ p=$0.3954$\\\\ [0.8ex]  \n",
      "GDP per capita & $-0.61^{**}$ & $0.19$ & $0.68$ & $0.44$\\\\ \n",
      " & $(-1.18,-0.03)$ & $(-1.08,1.46)$ & $(-0.34,1.7)$ & $(-0.25,1.12)$\\\\ \n",
      " & S.E. $0.29$ p=$0.038$ & S.E. $0.65$ p=$0.7655$ & S.E. $0.52$ p=$0.1911$ & S.E. $0.35$ p=$0.2121$\\\\ [0.8ex]  \n",
      "Number of publications &  & $-0.69^{*}$ & $-0.2$ & $-0.08$\\\\ \n",
      " &  & $(-1.39,0.02)$ & $(-0.75,0.34)$ & $(-0.5,0.34)$\\\\ \n",
      " &  & S.E. $0.36$ p=$0.0559$ & S.E. $0.28$ p=$0.4694$ & S.E. $0.21$ p=$0.7069$\\\\ [0.8ex]  \n",
      "Percent of GDP for R&D &  & $0.11$ & $0.0$ & $0.01$\\\\ \n",
      " &  & $(-0.09,0.31)$ & $(-0.13,0.13)$ & $(-0.12,0.14)$\\\\ \n",
      " &  & S.E. $0.1$ p=$0.2837$ & S.E. $0.07$ p=$0.9542$ & S.E. $0.07$ p=$0.898$\\\\ [0.8ex]  \n",
      "Fraction national authors &  &  & $-0.57^{***}$ & $-0.59^{***}$\\\\ \n",
      " &  &  & $(-0.75,-0.39)$ & $(-0.77,-0.42)$\\\\ \n",
      " &  &  & S.E. $0.09$ p=$0.0$ & S.E. $0.09$ p=$0.0$\\\\ [0.8ex]  \n",
      "National self-preference &  &  &  & $0.16^{**}$\\\\ \n",
      " &  &  &  & $(0.02,0.31)$\\\\ \n",
      " &  &  &  & S.E. $0.07$ p=$0.0283$\\\\ [0.8ex]  \n",
      "\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\textit{Note:} & \\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\ \n",
      "Observations & 1121 & 805 & 805 & 805 \\\\ \n",
      "$R^2$ & 0.0511 & 0.1511 & 0.347 & 0.3916 \\\\ \n",
      "Log Likelihood & -149.33 & -29.15 & 76.43 & 104.92 \\\\ \n",
      "F statistic & $27.43^{***}$ (d.f.=2) & $31.69^{***}$ (d.f.=4) & $75.56^{***}$ (d.f.=5) & $76.17^{***}$ (d.f.=6) \\\\ \n",
      "\\hline \n",
      "\\end{tabular} \n",
      "\\end{table}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\\\begin{table}[!htb] \\\\centering \\n    \\\\small\\n  \\\\caption{\\\\textbf{National self-preference.}\\n  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \\n  \\\\label{table:} \\n  \\\\begin{tabular}{lllll}\\\\\\\\[-1.8ex]\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\multicolumn{5}{c}{\\\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\\\\\[0.8ex]\\\\hline \\n & \\\\multicolumn{4}{c}{Model}  \\\\\\\\ \\\\cline{2-5} \\\\\\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\\\\\ \\\\hline \\\\\\\\[-1.8ex] \\nPopulation & $-1.41$ & $0.55$ & $-0.38$ & $-0.53$\\\\\\\\ \\n & $(-3.13,0.3)$ & $(-1.36,2.46)$ & $(-1.65,0.88)$ & $(-1.76,0.7)$\\\\\\\\ \\n & S.E. $0.88$ p=$0.1068$ & S.E. $0.97$ p=$0.5726$ & S.E. $0.64$ p=$0.5536$ & S.E. $0.63$ p=$0.3954$\\\\\\\\ [0.8ex]  \\nGDP per capita & $-0.61^{**}$ & $0.19$ & $0.68$ & $0.44$\\\\\\\\ \\n & $(-1.18,-0.03)$ & $(-1.08,1.46)$ & $(-0.34,1.7)$ & $(-0.25,1.12)$\\\\\\\\ \\n & S.E. $0.29$ p=$0.038$ & S.E. $0.65$ p=$0.7655$ & S.E. $0.52$ p=$0.1911$ & S.E. $0.35$ p=$0.2121$\\\\\\\\ [0.8ex]  \\nNumber of publications &  & $-0.69^{*}$ & $-0.2$ & $-0.08$\\\\\\\\ \\n &  & $(-1.39,0.02)$ & $(-0.75,0.34)$ & $(-0.5,0.34)$\\\\\\\\ \\n &  & S.E. $0.36$ p=$0.0559$ & S.E. $0.28$ p=$0.4694$ & S.E. $0.21$ p=$0.7069$\\\\\\\\ [0.8ex]  \\nPercent of GDP for R&D &  & $0.11$ & $0.0$ & $0.01$\\\\\\\\ \\n &  & $(-0.09,0.31)$ & $(-0.13,0.13)$ & $(-0.12,0.14)$\\\\\\\\ \\n &  & S.E. $0.1$ p=$0.2837$ & S.E. $0.07$ p=$0.9542$ & S.E. $0.07$ p=$0.898$\\\\\\\\ [0.8ex]  \\nFraction national authors &  &  & $-0.57^{***}$ & $-0.59^{***}$\\\\\\\\ \\n &  &  & $(-0.75,-0.39)$ & $(-0.77,-0.42)$\\\\\\\\ \\n &  &  & S.E. $0.09$ p=$0.0$ & S.E. $0.09$ p=$0.0$\\\\\\\\ [0.8ex]  \\nNational self-preference &  &  &  & $0.16^{**}$\\\\\\\\ \\n &  &  &  & $(0.02,0.31)$\\\\\\\\ \\n &  &  &  & S.E. $0.07$ p=$0.0283$\\\\\\\\ [0.8ex]  \\n\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\textit{Note:} & \\\\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\\\\\ \\nObservations & 1121 & 805 & 805 & 805 \\\\\\\\ \\n$R^2$ & 0.0511 & 0.1511 & 0.347 & 0.3916 \\\\\\\\ \\nLog Likelihood & -149.33 & -29.15 & 76.43 & 104.92 \\\\\\\\ \\nF statistic & $27.43^{***}$ (d.f.=2) & $31.69^{***}$ (d.f.=4) & $75.56^{***}$ (d.f.=5) & $76.17^{***}$ (d.f.=6) \\\\\\\\ \\n\\\\hline \\n\\\\end{tabular} \\n\\\\end{table}'"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "namedict['AUC'] = 'National self-preference'\n",
    "\n",
    "effect_vars = ['Country', 'Year']\n",
    "exog_vars = ['Pop', 'GDP_PCAP', \"NumPub\", 'RND_per', \"FractionNationalAuthors\", \"AUC\"]\n",
    "dep_var = \"NormedTop\"\n",
    "\n",
    "self_panel = self_auc[effect_vars + exog_vars + [dep_var]].dropna().reset_index(drop=True)\n",
    "self_panel = self_panel[self_panel['Year'] >= 1990]\n",
    "for c in exog_vars + [dep_var]:\n",
    "    self_panel[c] = (self_panel[c] - self_panel[c].mean())/self_panel[c].std()\n",
    "\n",
    "self_panel=self_panel.set_index(effect_vars)\n",
    "\n",
    "#exog = sm.add_constant(self_panel[exog_vars])\n",
    "mod = PanelOLS(self_panel[dep_var], self_panel[exog_vars], entity_effects=True, time_effects=True)\n",
    "res = mod.fit(cov_type='clustered', cluster_entity=True)\n",
    "print(res)\n",
    "\n",
    "results_as_html = res.summary.tables[1].as_html()\n",
    "sumdf_topfrac = pd.read_html(results_as_html, header=0, index_col=0)[0]\n",
    "\n",
    "exog_var_sets = [['Pop', 'GDP_PCAP'], [\"NumPub\", 'RND_per'], ['FractionNationalAuthors'], ['AUC']]\n",
    "dep_var = \"NormedTop\"\n",
    "\n",
    "make_latex_table(data=self_auc, effect_vars = effect_vars, use_const = False,\n",
    "                 exog_var_sets = exog_var_sets, dep_var = dep_var, namedict = namedict)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "d91b9290",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          PanelOLS Estimation Summary                           \n",
      "================================================================================\n",
      "Dep. Variable:        TopicDiversity2   R-squared:                        0.1860\n",
      "Estimator:                   PanelOLS   R-squared (Between):             -0.1184\n",
      "No. Observations:                 805   R-squared (Within):              -0.8011\n",
      "Date:                Thu, Jan 25 2024   R-squared (Overall):             -1.1839\n",
      "Time:                        12:51:00   Log-likelihood                    360.98\n",
      "Cov. Estimator:             Clustered                                           \n",
      "                                        F-statistic:                      27.042\n",
      "Entities:                          68   P-value                           0.0000\n",
      "Avg Obs:                       11.838   Distribution:                   F(6,710)\n",
      "Min Obs:                       1.0000                                           \n",
      "Max Obs:                       22.000   F-statistic (robust):             4.0934\n",
      "                                        P-value                           0.0005\n",
      "Time periods:                      22   Distribution:                   F(6,710)\n",
      "Avg Obs:                       36.591                                           \n",
      "Min Obs:                       18.000                                           \n",
      "Max Obs:                       53.000                                           \n",
      "                                                                                \n",
      "                                    Parameter Estimates                                    \n",
      "===========================================================================================\n",
      "                         Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
      "-------------------------------------------------------------------------------------------\n",
      "Pop                        -1.2566     0.8162    -1.5396     0.1241     -2.8590      0.3458\n",
      "GDP_PCAP                   -0.1163     0.2556    -0.4550     0.6492     -0.6182      0.3856\n",
      "NumPub                      0.0407     0.1203     0.3382     0.7353     -0.1955      0.2769\n",
      "RND_per                    -0.0497     0.0690    -0.7202     0.4716     -0.1853      0.0858\n",
      "FractionNationalAuthors     0.1606     0.0702     2.2893     0.0224      0.0229      0.2984\n",
      "AUC                        -0.1422     0.0553    -2.5700     0.0104     -0.2508     -0.0336\n",
      "===========================================================================================\n",
      "\n",
      "F-test for Poolability: 229.81\n",
      "P-value: 0.0000\n",
      "Distribution: F(88,710)\n",
      "\n",
      "Included effects: Entity, Time\n",
      "\\begin{table}[!htb] \\centering \n",
      "    \\small\n",
      "  \\caption{\\textbf{National self-preference.}\n",
      "  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \n",
      "  \\label{table:} \n",
      "  \\begin{tabular}{lllll}\\\\[-1.8ex]\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\multicolumn{5}{c}{\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\[0.8ex]\\hline \n",
      " & \\multicolumn{4}{c}{Model}  \\\\ \\cline{2-5} \\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\ \\hline \\\\[-1.8ex] \n",
      "Population & $-0.7$ & $-1.62^{*}$ & $-1.39^{*}$ & $-1.26$\\\\ \n",
      " & $(-1.76,0.37)$ & $(-3.32,0.08)$ & $(-3.03,0.25)$ & $(-2.86,0.35)$\\\\ \n",
      " & S.E. $0.54$ p=$0.1987$ & S.E. $0.87$ p=$0.0625$ & S.E. $0.84$ p=$0.0973$ & S.E. $0.82$ p=$0.1241$\\\\ [0.8ex]  \n",
      "GDP per capita & $0.06$ & $-0.21$ & $-0.33$ & $-0.12$\\\\ \n",
      " & $(-0.43,0.55)$ & $(-0.91,0.5)$ & $(-1.01,0.35)$ & $(-0.62,0.39)$\\\\ \n",
      " & S.E. $0.25$ p=$0.8087$ & S.E. $0.36$ p=$0.5677$ & S.E. $0.35$ p=$0.3447$ & S.E. $0.26$ p=$0.6492$\\\\ [0.8ex]  \n",
      "Number of publications &  & $0.27^{***}$ & $0.14$ & $0.04$\\\\ \n",
      " &  & $(0.08,0.45)$ & $(-0.07,0.36)$ & $(-0.2,0.28)$\\\\ \n",
      " &  & S.E. $0.1$ p=$0.0057$ & S.E. $0.11$ p=$0.1906$ & S.E. $0.12$ p=$0.7353$\\\\ [0.8ex]  \n",
      "Percent of GDP for R&D &  & $-0.07$ & $-0.05$ & $-0.05$\\\\ \n",
      " &  & $(-0.23,0.09)$ & $(-0.2,0.11)$ & $(-0.19,0.09)$\\\\ \n",
      " &  & S.E. $0.08$ p=$0.3696$ & S.E. $0.08$ p=$0.5617$ & S.E. $0.07$ p=$0.4716$\\\\ [0.8ex]  \n",
      "Fraction national authors &  &  & $0.14^{**}$ & $0.16^{**}$\\\\ \n",
      " &  &  & $(0.0,0.28)$ & $(0.02,0.3)$\\\\ \n",
      " &  &  & S.E. $0.07$ p=$0.0438$ & S.E. $0.07$ p=$0.0224$\\\\ [0.8ex]  \n",
      "National self-preference &  &  &  & $-0.14^{**}$\\\\ \n",
      " &  &  &  & $(-0.25,-0.03)$\\\\ \n",
      " &  &  &  & S.E. $0.06$ p=$0.0104$\\\\ [0.8ex]  \n",
      "\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\textit{Note:} & \\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\ \n",
      "Observations & 1121 & 805 & 805 & 805 \\\\ \n",
      "$R^2$ & 0.0164 & 0.071 & 0.1017 & 0.186 \\\\ \n",
      "Log Likelihood & 550.61 & 307.79 & 321.32 & 360.98 \\\\ \n",
      "F statistic & $8.47^{***}$ (d.f.=2) & $13.61^{***}$ (d.f.=4) & $16.1^{***}$ (d.f.=5) & $27.04^{***}$ (d.f.=6) \\\\ \n",
      "\\hline \n",
      "\\end{tabular} \n",
      "\\end{table}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\\\begin{table}[!htb] \\\\centering \\n    \\\\small\\n  \\\\caption{\\\\textbf{National self-preference.}\\n  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \\n  \\\\label{table:} \\n  \\\\begin{tabular}{lllll}\\\\\\\\[-1.8ex]\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\multicolumn{5}{c}{\\\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\\\\\[0.8ex]\\\\hline \\n & \\\\multicolumn{4}{c}{Model}  \\\\\\\\ \\\\cline{2-5} \\\\\\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\\\\\ \\\\hline \\\\\\\\[-1.8ex] \\nPopulation & $-0.7$ & $-1.62^{*}$ & $-1.39^{*}$ & $-1.26$\\\\\\\\ \\n & $(-1.76,0.37)$ & $(-3.32,0.08)$ & $(-3.03,0.25)$ & $(-2.86,0.35)$\\\\\\\\ \\n & S.E. $0.54$ p=$0.1987$ & S.E. $0.87$ p=$0.0625$ & S.E. $0.84$ p=$0.0973$ & S.E. $0.82$ p=$0.1241$\\\\\\\\ [0.8ex]  \\nGDP per capita & $0.06$ & $-0.21$ & $-0.33$ & $-0.12$\\\\\\\\ \\n & $(-0.43,0.55)$ & $(-0.91,0.5)$ & $(-1.01,0.35)$ & $(-0.62,0.39)$\\\\\\\\ \\n & S.E. $0.25$ p=$0.8087$ & S.E. $0.36$ p=$0.5677$ & S.E. $0.35$ p=$0.3447$ & S.E. $0.26$ p=$0.6492$\\\\\\\\ [0.8ex]  \\nNumber of publications &  & $0.27^{***}$ & $0.14$ & $0.04$\\\\\\\\ \\n &  & $(0.08,0.45)$ & $(-0.07,0.36)$ & $(-0.2,0.28)$\\\\\\\\ \\n &  & S.E. $0.1$ p=$0.0057$ & S.E. $0.11$ p=$0.1906$ & S.E. $0.12$ p=$0.7353$\\\\\\\\ [0.8ex]  \\nPercent of GDP for R&D &  & $-0.07$ & $-0.05$ & $-0.05$\\\\\\\\ \\n &  & $(-0.23,0.09)$ & $(-0.2,0.11)$ & $(-0.19,0.09)$\\\\\\\\ \\n &  & S.E. $0.08$ p=$0.3696$ & S.E. $0.08$ p=$0.5617$ & S.E. $0.07$ p=$0.4716$\\\\\\\\ [0.8ex]  \\nFraction national authors &  &  & $0.14^{**}$ & $0.16^{**}$\\\\\\\\ \\n &  &  & $(0.0,0.28)$ & $(0.02,0.3)$\\\\\\\\ \\n &  &  & S.E. $0.07$ p=$0.0438$ & S.E. $0.07$ p=$0.0224$\\\\\\\\ [0.8ex]  \\nNational self-preference &  &  &  & $-0.14^{**}$\\\\\\\\ \\n &  &  &  & $(-0.25,-0.03)$\\\\\\\\ \\n &  &  &  & S.E. $0.06$ p=$0.0104$\\\\\\\\ [0.8ex]  \\n\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\textit{Note:} & \\\\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\\\\\ \\nObservations & 1121 & 805 & 805 & 805 \\\\\\\\ \\n$R^2$ & 0.0164 & 0.071 & 0.1017 & 0.186 \\\\\\\\ \\nLog Likelihood & 550.61 & 307.79 & 321.32 & 360.98 \\\\\\\\ \\nF statistic & $8.47^{***}$ (d.f.=2) & $13.61^{***}$ (d.f.=4) & $16.1^{***}$ (d.f.=5) & $27.04^{***}$ (d.f.=6) \\\\\\\\ \\n\\\\hline \\n\\\\end{tabular} \\n\\\\end{table}'"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "effect_vars = ['Country', 'Year']\n",
    "exog_vars = ['Pop', 'GDP_PCAP', \"NumPub\", 'RND_per',  \"FractionNationalAuthors\", \"AUC\"]\n",
    "dep_var = \"TopicDiversity2\"\n",
    "\n",
    "self_panel = self_auc[effect_vars + exog_vars + [dep_var]].dropna().reset_index(drop=True)\n",
    "self_panel = self_panel[self_panel['Year'] >= 1990]\n",
    "\n",
    "for c in exog_vars + [dep_var]:\n",
    "    self_panel[c] = (self_panel[c] - self_panel[c].mean())/self_panel[c].std()\n",
    "    \n",
    "self_panel=self_panel.set_index(effect_vars)\n",
    "\n",
    "#exog = sm.add_constant(self_panel[exog_vars])\n",
    "mod = PanelOLS(self_panel[dep_var], self_panel[exog_vars], entity_effects=True, time_effects=True)\n",
    "res = mod.fit(cov_type='clustered', cluster_entity=True)\n",
    "print(res)\n",
    "\n",
    "results_as_html = res.summary.tables[1].as_html()\n",
    "sumdf_div = pd.read_html(results_as_html, header=0, index_col=0)[0]\n",
    "\n",
    "make_latex_table(data=self_auc, effect_vars = effect_vars, use_const = False,\n",
    "                 exog_var_sets = exog_var_sets, dep_var = dep_var, namedict = namedict)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "fd78aba3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          PanelOLS Estimation Summary                           \n",
      "================================================================================\n",
      "Dep. Variable:                PAT_res   R-squared:                        0.5953\n",
      "Estimator:                   PanelOLS   R-squared (Between):              0.6455\n",
      "No. Observations:                 766   R-squared (Within):               0.2194\n",
      "Date:                Thu, Jan 25 2024   R-squared (Overall):              0.5636\n",
      "Time:                        13:41:36   Log-likelihood                    504.08\n",
      "Cov. Estimator:             Clustered                                           \n",
      "                                        F-statistic:                      165.46\n",
      "Entities:                          64   P-value                           0.0000\n",
      "Avg Obs:                       11.969   Distribution:                   F(6,675)\n",
      "Min Obs:                       1.0000                                           \n",
      "Max Obs:                       22.000   F-statistic (robust):             21.158\n",
      "                                        P-value                           0.0000\n",
      "Time periods:                      22   Distribution:                   F(6,675)\n",
      "Avg Obs:                       34.818                                           \n",
      "Min Obs:                       17.000                                           \n",
      "Max Obs:                       53.000                                           \n",
      "                                                                                \n",
      "                                    Parameter Estimates                                    \n",
      "===========================================================================================\n",
      "                         Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
      "-------------------------------------------------------------------------------------------\n",
      "Pop                         0.1579     0.7961     0.1983     0.8428     -1.4053      1.7211\n",
      "GDP_PCAP                    0.5912     0.2293     2.5777     0.0102      0.1409      1.0415\n",
      "NumPub                      0.4740     0.1385     3.4225     0.0007      0.2021      0.7459\n",
      "RND_per                     0.1038     0.0620     1.6727     0.0949     -0.0180      0.2255\n",
      "FractionNationalAuthors     0.0565     0.0771     0.7326     0.4641     -0.0949      0.2079\n",
      "AUC                         0.0737     0.0356     2.0713     0.0387      0.0038      0.1436\n",
      "===========================================================================================\n",
      "\n",
      "F-test for Poolability: 72.354\n",
      "P-value: 0.0000\n",
      "Distribution: F(84,675)\n",
      "\n",
      "Included effects: Entity, Time\n",
      "\\begin{table}[!htb] \\centering \n",
      "    \\small\n",
      "  \\caption{\\textbf{National self-preference.}\n",
      "  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \n",
      "  \\label{table:} \n",
      "  \\begin{tabular}{lllll}\\\\[-1.8ex]\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\multicolumn{5}{c}{\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\[0.8ex]\\hline \n",
      " & \\multicolumn{4}{c}{Model}  \\\\ \\cline{2-5} \\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\ \\hline \\\\[-1.8ex] \n",
      "Population & $1.29^{*}$ & $0.11$ & $0.22$ & $0.16$\\\\ \n",
      " & $(-0.16,2.74)$ & $(-1.35,1.57)$ & $(-1.32,1.76)$ & $(-1.41,1.72)$\\\\ \n",
      " & S.E. $0.74$ p=$0.0821$ & S.E. $0.74$ p=$0.8811$ & S.E. $0.78$ p=$0.7819$ & S.E. $0.8$ p=$0.8428$\\\\ [0.8ex]  \n",
      "GDP per capita & $1.33^{***}$ & $0.76^{***}$ & $0.71^{**}$ & $0.59^{**}$\\\\ \n",
      " & $(0.89,1.77)$ & $(0.25,1.27)$ & $(0.13,1.29)$ & $(0.14,1.04)$\\\\ \n",
      " & S.E. $0.22$ p=$0.0$ & S.E. $0.26$ p=$0.0034$ & S.E. $0.3$ p=$0.0166$ & S.E. $0.23$ p=$0.0102$\\\\ [0.8ex]  \n",
      "Number of publications &  & $0.47^{***}$ & $0.42^{***}$ & $0.47^{***}$\\\\ \n",
      " &  & $(0.22,0.73)$ & $(0.15,0.69)$ & $(0.2,0.75)$\\\\ \n",
      " &  & S.E. $0.13$ p=$0.0003$ & S.E. $0.14$ p=$0.0022$ & S.E. $0.14$ p=$0.0007$\\\\ [0.8ex]  \n",
      "Percent of GDP for R&D &  & $0.09$ & $0.1$ & $0.1^{*}$\\\\ \n",
      " &  & $(-0.03,0.21)$ & $(-0.03,0.23)$ & $(-0.02,0.23)$\\\\ \n",
      " &  & S.E. $0.06$ p=$0.1429$ & S.E. $0.07$ p=$0.1371$ & S.E. $0.06$ p=$0.0949$\\\\ [0.8ex]  \n",
      "Fraction national authors &  &  & $0.06$ & $0.06$\\\\ \n",
      " &  &  & $(-0.09,0.21)$ & $(-0.09,0.21)$\\\\ \n",
      " &  &  & S.E. $0.08$ p=$0.4076$ & S.E. $0.08$ p=$0.4641$\\\\ [0.8ex]  \n",
      "National self-preference &  &  &  & $0.07^{**}$\\\\ \n",
      " &  &  &  & $(0.0,0.14)$\\\\ \n",
      " &  &  &  & S.E. $0.04$ p=$0.0387$\\\\ [0.8ex]  \n",
      "\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\textit{Note:} & \\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\ \n",
      "Observations & 1029 & 766 & 766 & 766 \\\\ \n",
      "$R^2$ & 0.3614 & 0.5735 & 0.5783 & 0.5953 \\\\ \n",
      "Log Likelihood & 464.88 & 484.01 & 488.33 & 504.08 \\\\ \n",
      "F statistic & $262.89^{***}$ (d.f.=2) & $227.58^{***}$ (d.f.=4) & $185.39^{***}$ (d.f.=5) & $165.46^{***}$ (d.f.=6) \\\\ \n",
      "\\hline \n",
      "\\end{tabular} \n",
      "\\end{table}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\\\begin{table}[!htb] \\\\centering \\n    \\\\small\\n  \\\\caption{\\\\textbf{National self-preference.}\\n  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \\n  \\\\label{table:} \\n  \\\\begin{tabular}{lllll}\\\\\\\\[-1.8ex]\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\multicolumn{5}{c}{\\\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\\\\\[0.8ex]\\\\hline \\n & \\\\multicolumn{4}{c}{Model}  \\\\\\\\ \\\\cline{2-5} \\\\\\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\\\\\ \\\\hline \\\\\\\\[-1.8ex] \\nPopulation & $1.29^{*}$ & $0.11$ & $0.22$ & $0.16$\\\\\\\\ \\n & $(-0.16,2.74)$ & $(-1.35,1.57)$ & $(-1.32,1.76)$ & $(-1.41,1.72)$\\\\\\\\ \\n & S.E. $0.74$ p=$0.0821$ & S.E. $0.74$ p=$0.8811$ & S.E. $0.78$ p=$0.7819$ & S.E. $0.8$ p=$0.8428$\\\\\\\\ [0.8ex]  \\nGDP per capita & $1.33^{***}$ & $0.76^{***}$ & $0.71^{**}$ & $0.59^{**}$\\\\\\\\ \\n & $(0.89,1.77)$ & $(0.25,1.27)$ & $(0.13,1.29)$ & $(0.14,1.04)$\\\\\\\\ \\n & S.E. $0.22$ p=$0.0$ & S.E. $0.26$ p=$0.0034$ & S.E. $0.3$ p=$0.0166$ & S.E. $0.23$ p=$0.0102$\\\\\\\\ [0.8ex]  \\nNumber of publications &  & $0.47^{***}$ & $0.42^{***}$ & $0.47^{***}$\\\\\\\\ \\n &  & $(0.22,0.73)$ & $(0.15,0.69)$ & $(0.2,0.75)$\\\\\\\\ \\n &  & S.E. $0.13$ p=$0.0003$ & S.E. $0.14$ p=$0.0022$ & S.E. $0.14$ p=$0.0007$\\\\\\\\ [0.8ex]  \\nPercent of GDP for R&D &  & $0.09$ & $0.1$ & $0.1^{*}$\\\\\\\\ \\n &  & $(-0.03,0.21)$ & $(-0.03,0.23)$ & $(-0.02,0.23)$\\\\\\\\ \\n &  & S.E. $0.06$ p=$0.1429$ & S.E. $0.07$ p=$0.1371$ & S.E. $0.06$ p=$0.0949$\\\\\\\\ [0.8ex]  \\nFraction national authors &  &  & $0.06$ & $0.06$\\\\\\\\ \\n &  &  & $(-0.09,0.21)$ & $(-0.09,0.21)$\\\\\\\\ \\n &  &  & S.E. $0.08$ p=$0.4076$ & S.E. $0.08$ p=$0.4641$\\\\\\\\ [0.8ex]  \\nNational self-preference &  &  &  & $0.07^{**}$\\\\\\\\ \\n &  &  &  & $(0.0,0.14)$\\\\\\\\ \\n &  &  &  & S.E. $0.04$ p=$0.0387$\\\\\\\\ [0.8ex]  \\n\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\textit{Note:} & \\\\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\\\\\ \\nObservations & 1029 & 766 & 766 & 766 \\\\\\\\ \\n$R^2$ & 0.3614 & 0.5735 & 0.5783 & 0.5953 \\\\\\\\ \\nLog Likelihood & 464.88 & 484.01 & 488.33 & 504.08 \\\\\\\\ \\nF statistic & $262.89^{***}$ (d.f.=2) & $227.58^{***}$ (d.f.=4) & $185.39^{***}$ (d.f.=5) & $165.46^{***}$ (d.f.=6) \\\\\\\\ \\n\\\\hline \\n\\\\end{tabular} \\n\\\\end{table}'"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "effect_vars = ['Country', 'Year']\n",
    "exog_vars = ['Pop', 'GDP_PCAP', \"NumPub\", 'RND_per',  \"FractionNationalAuthors\", \"AUC\"]\n",
    "dep_var = \"PAT_res\"\n",
    "\n",
    "self_panel = self_auc[effect_vars + exog_vars + [dep_var]].dropna().reset_index(drop=True)\n",
    "self_panel = self_panel[self_panel['Year'] >= 1990]\n",
    "\n",
    "for c in exog_vars + [dep_var]:\n",
    "    self_panel[c] = (self_panel[c] - self_panel[c].mean())/self_panel[c].std()\n",
    "    \n",
    "self_panel=self_panel.set_index(effect_vars)\n",
    "\n",
    "#exog = sm.add_constant(self_panel[exog_vars])\n",
    "mod = PanelOLS(self_panel[dep_var], self_panel[exog_vars], entity_effects=True, time_effects=True)\n",
    "res = mod.fit(cov_type='clustered', cluster_entity=True)\n",
    "print(res)\n",
    "\n",
    "results_as_html = res.summary.tables[1].as_html()\n",
    "sumdf_pat = pd.read_html(results_as_html, header=0, index_col=0)[0]\n",
    "\n",
    "make_latex_table(data=self_auc, effect_vars = effect_vars, use_const = False,\n",
    "                 exog_var_sets = exog_var_sets, dep_var = dep_var, namedict = namedict)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "2dbf356b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          PanelOLS Estimation Summary                           \n",
      "================================================================================\n",
      "Dep. Variable:              PAT_total   R-squared:                        0.4488\n",
      "Estimator:                   PanelOLS   R-squared (Between):              0.1157\n",
      "No. Observations:                 765   R-squared (Within):               0.1909\n",
      "Date:                Thu, Jan 25 2024   R-squared (Overall):             -0.3219\n",
      "Time:                        13:41:36   Log-likelihood                    444.40\n",
      "Cov. Estimator:             Clustered                                           \n",
      "                                        F-statistic:                      91.448\n",
      "Entities:                          64   P-value                           0.0000\n",
      "Avg Obs:                       11.953   Distribution:                   F(6,674)\n",
      "Min Obs:                       1.0000                                           \n",
      "Max Obs:                       22.000   F-statistic (robust):             13.163\n",
      "                                        P-value                           0.0000\n",
      "Time periods:                      22   Distribution:                   F(6,674)\n",
      "Avg Obs:                       34.773                                           \n",
      "Min Obs:                       17.000                                           \n",
      "Max Obs:                       53.000                                           \n",
      "                                                                                \n",
      "                                    Parameter Estimates                                    \n",
      "===========================================================================================\n",
      "                         Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
      "-------------------------------------------------------------------------------------------\n",
      "Pop                        -0.1000     0.7019    -0.1424     0.8868     -1.4782      1.2783\n",
      "GDP_PCAP                    0.7512     0.2610     2.8777     0.0041      0.2386      1.2637\n",
      "NumPub                      0.2557     0.1895     1.3497     0.1776     -0.1163      0.6278\n",
      "RND_per                     0.0729     0.0724     1.0065     0.3145     -0.0693      0.2151\n",
      "FractionNationalAuthors     0.0409     0.0584     0.7010     0.4836     -0.0737      0.1555\n",
      "AUC                         0.0561     0.0389     1.4438     0.1493     -0.0202      0.1325\n",
      "===========================================================================================\n",
      "\n",
      "F-test for Poolability: 85.712\n",
      "P-value: 0.0000\n",
      "Distribution: F(84,674)\n",
      "\n",
      "Included effects: Entity, Time\n",
      "\\begin{table}[!htb] \\centering \n",
      "    \\small\n",
      "  \\caption{\\textbf{National self-preference.}\n",
      "  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \n",
      "  \\label{table:} \n",
      "  \\begin{tabular}{lllll}\\\\[-1.8ex]\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\multicolumn{5}{c}{\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\[0.8ex]\\hline \n",
      " & \\multicolumn{4}{c}{Model}  \\\\ \\cline{2-5} \\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\ \\hline \\\\[-1.8ex] \n",
      "Population & $1.12$ & $-0.11$ & $-0.03$ & $-0.1$\\\\ \n",
      " & $(-0.33,2.57)$ & $(-1.5,1.28)$ & $(-1.4,1.34)$ & $(-1.48,1.28)$\\\\ \n",
      " & S.E. $0.74$ p=$0.1304$ & S.E. $0.71$ p=$0.8751$ & S.E. $0.7$ p=$0.9678$ & S.E. $0.7$ p=$0.8868$\\\\ [0.8ex]  \n",
      "GDP per capita & $1.18^{***}$ & $0.88^{***}$ & $0.84^{***}$ & $0.75^{***}$\\\\ \n",
      " & $(0.64,1.72)$ & $(0.33,1.43)$ & $(0.26,1.43)$ & $(0.24,1.26)$\\\\ \n",
      " & S.E. $0.27$ p=$0.0$ & S.E. $0.28$ p=$0.0017$ & S.E. $0.3$ p=$0.0049$ & S.E. $0.26$ p=$0.0041$\\\\ [0.8ex]  \n",
      "Number of publications &  & $0.25$ & $0.21$ & $0.26$\\\\ \n",
      " &  & $(-0.06,0.57)$ & $(-0.14,0.56)$ & $(-0.12,0.63)$\\\\ \n",
      " &  & S.E. $0.16$ p=$0.1157$ & S.E. $0.18$ p=$0.2285$ & S.E. $0.19$ p=$0.1776$\\\\ [0.8ex]  \n",
      "Percent of GDP for R&D &  & $0.06$ & $0.07$ & $0.07$\\\\ \n",
      " &  & $(-0.08,0.2)$ & $(-0.08,0.21)$ & $(-0.07,0.22)$\\\\ \n",
      " &  & S.E. $0.07$ p=$0.3886$ & S.E. $0.07$ p=$0.3736$ & S.E. $0.07$ p=$0.3145$\\\\ [0.8ex]  \n",
      "Fraction national authors &  &  & $0.05$ & $0.04$\\\\ \n",
      " &  &  & $(-0.07,0.16)$ & $(-0.07,0.16)$\\\\ \n",
      " &  &  & S.E. $0.06$ p=$0.4103$ & S.E. $0.06$ p=$0.4836$\\\\ [0.8ex]  \n",
      "National self-preference &  &  &  & $0.06$\\\\ \n",
      " &  &  &  & $(-0.02,0.13)$\\\\ \n",
      " &  &  &  & S.E. $0.04$ p=$0.1493$\\\\ [0.8ex]  \n",
      "\\hline \n",
      "\\hline \\\\[-1.8ex] \n",
      "\\textit{Note:} & \\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\ \n",
      "Observations & 1026 & 765 & 765 & 765 \\\\ \n",
      "$R^2$ & 0.2816 & 0.4344 & 0.4376 & 0.4488 \\\\ \n",
      "Log Likelihood & 394.63 & 434.6 & 436.73 & 444.4 \\\\ \n",
      "F statistic & $181.45^{***}$ (d.f.=2) & $129.82^{***}$ (d.f.=4) & $105.04^{***}$ (d.f.=5) & $91.45^{***}$ (d.f.=6) \\\\ \n",
      "\\hline \n",
      "\\end{tabular} \n",
      "\\end{table}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\\\begin{table}[!htb] \\\\centering \\n    \\\\small\\n  \\\\caption{\\\\textbf{National self-preference.}\\n  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \\n  \\\\label{table:} \\n  \\\\begin{tabular}{lllll}\\\\\\\\[-1.8ex]\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\multicolumn{5}{c}{\\\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\\\\\[0.8ex]\\\\hline \\n & \\\\multicolumn{4}{c}{Model}  \\\\\\\\ \\\\cline{2-5} \\\\\\\\[-1.8ex] & (1)& (2)& (3)& (4) \\\\\\\\ \\\\hline \\\\\\\\[-1.8ex] \\nPopulation & $1.12$ & $-0.11$ & $-0.03$ & $-0.1$\\\\\\\\ \\n & $(-0.33,2.57)$ & $(-1.5,1.28)$ & $(-1.4,1.34)$ & $(-1.48,1.28)$\\\\\\\\ \\n & S.E. $0.74$ p=$0.1304$ & S.E. $0.71$ p=$0.8751$ & S.E. $0.7$ p=$0.9678$ & S.E. $0.7$ p=$0.8868$\\\\\\\\ [0.8ex]  \\nGDP per capita & $1.18^{***}$ & $0.88^{***}$ & $0.84^{***}$ & $0.75^{***}$\\\\\\\\ \\n & $(0.64,1.72)$ & $(0.33,1.43)$ & $(0.26,1.43)$ & $(0.24,1.26)$\\\\\\\\ \\n & S.E. $0.27$ p=$0.0$ & S.E. $0.28$ p=$0.0017$ & S.E. $0.3$ p=$0.0049$ & S.E. $0.26$ p=$0.0041$\\\\\\\\ [0.8ex]  \\nNumber of publications &  & $0.25$ & $0.21$ & $0.26$\\\\\\\\ \\n &  & $(-0.06,0.57)$ & $(-0.14,0.56)$ & $(-0.12,0.63)$\\\\\\\\ \\n &  & S.E. $0.16$ p=$0.1157$ & S.E. $0.18$ p=$0.2285$ & S.E. $0.19$ p=$0.1776$\\\\\\\\ [0.8ex]  \\nPercent of GDP for R&D &  & $0.06$ & $0.07$ & $0.07$\\\\\\\\ \\n &  & $(-0.08,0.2)$ & $(-0.08,0.21)$ & $(-0.07,0.22)$\\\\\\\\ \\n &  & S.E. $0.07$ p=$0.3886$ & S.E. $0.07$ p=$0.3736$ & S.E. $0.07$ p=$0.3145$\\\\\\\\ [0.8ex]  \\nFraction national authors &  &  & $0.05$ & $0.04$\\\\\\\\ \\n &  &  & $(-0.07,0.16)$ & $(-0.07,0.16)$\\\\\\\\ \\n &  &  & S.E. $0.06$ p=$0.4103$ & S.E. $0.06$ p=$0.4836$\\\\\\\\ [0.8ex]  \\nNational self-preference &  &  &  & $0.06$\\\\\\\\ \\n &  &  &  & $(-0.02,0.13)$\\\\\\\\ \\n &  &  &  & S.E. $0.04$ p=$0.1493$\\\\\\\\ [0.8ex]  \\n\\\\hline \\n\\\\hline \\\\\\\\[-1.8ex] \\n\\\\textit{Note:} & \\\\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\\\\\ \\nObservations & 1026 & 765 & 765 & 765 \\\\\\\\ \\n$R^2$ & 0.2816 & 0.4344 & 0.4376 & 0.4488 \\\\\\\\ \\nLog Likelihood & 394.63 & 434.6 & 436.73 & 444.4 \\\\\\\\ \\nF statistic & $181.45^{***}$ (d.f.=2) & $129.82^{***}$ (d.f.=4) & $105.04^{***}$ (d.f.=5) & $91.45^{***}$ (d.f.=6) \\\\\\\\ \\n\\\\hline \\n\\\\end{tabular} \\n\\\\end{table}'"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "effect_vars = ['Country', 'Year']\n",
    "exog_vars = ['Pop', 'GDP_PCAP', \"NumPub\", 'RND_per',  \"FractionNationalAuthors\", \"AUC\"]\n",
    "dep_var = \"PAT_total\"\n",
    "\n",
    "self_panel = self_auc[effect_vars + exog_vars + [dep_var]].dropna().reset_index(drop=True)\n",
    "self_panel = self_panel[self_panel['Year'] >= 1990]\n",
    "\n",
    "for c in exog_vars + [dep_var]:\n",
    "    self_panel[c] = (self_panel[c] - self_panel[c].mean())/self_panel[c].std()\n",
    "    \n",
    "self_panel=self_panel.set_index(effect_vars)\n",
    "\n",
    "#exog = sm.add_constant(self_panel[exog_vars])\n",
    "mod = PanelOLS(self_panel[dep_var], self_panel[exog_vars], entity_effects=True, time_effects=True)\n",
    "res = mod.fit(cov_type='clustered', cluster_entity=True)\n",
    "print(res)\n",
    "\n",
    "results_as_html = res.summary.tables[1].as_html()\n",
    "sumdf_pattot = pd.read_html(results_as_html, header=0, index_col=0)[0]\n",
    "\n",
    "make_latex_table(data=self_auc, effect_vars = effect_vars, use_const = False,\n",
    "                 exog_var_sets = exog_var_sets, dep_var = dep_var, namedict = namedict)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "2514e3e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 172308 characters]\n",
      "text/plain": [
       "<Figure size 1000x1000 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "layout = [\n",
    "    [\"A\", \"A\", \"A\", \"A\", \"A\", \"A\"],\n",
    "    [ \"B\", \"B\", \"B\", \"C\",\"C\",\"C\"],\n",
    "    [ \"D\", \"D\", 'E','E','F', 'F'],\n",
    "    [ \"D\", \"D\", 'E','E', 'F', 'F']\n",
    "]\n",
    "\n",
    "fig, axd = plt.subplot_mosaic(layout, figsize=(10,10))\n",
    "\n",
    "\n",
    "self_auc2017 = self_auc[self_auc['Year'] == 2017]\n",
    "line_stick_auc(axd[\"A\"], self_auc2017, ['US', 'CN', 'JP', 'CH', 'GB', 'BR'])\n",
    "\n",
    "countries2highlight = sorted(['US', 'CH', 'GB', 'DE', 'CN', 'IN', 'NP', 'ZA', 'TR', 'NG'])\n",
    "\n",
    "#for i,c in enumerate(['GB', 'JP', 'CN', 'CH', 'US']):\n",
    "for iax, countrylist in zip([\"B\", \"C\", \"D\"], [comm2country[2] + comm2country[3], comm2country[0] + comm2country[4]]):\n",
    "    subcountries2highlight = list(set(countries2highlight).intersection(set(countrylist)))\n",
    "    for c in countrylist:\n",
    "        selfcountry = self_auc[self_auc['Country'] == c]\n",
    "        selfcountry = selfcountry[selfcountry['Significant']]\n",
    "        selfcountry = selfcountry[selfcountry['Year'] >= 1970]\n",
    "        if selfcountry.shape[0] > 2 and c!= 'ET':\n",
    "            #print(c, selfcountry['AUC'].max())\n",
    "            if c in countries2highlight:\n",
    "                color = sns.color_palette(\"rocket\", n_colors = len(subcountries2highlight))[subcountries2highlight.index(c)]\n",
    "                axd[iax].plot(selfcountry['Year'].values, selfcountry['AUC'].values, color=color, lw=2, zorder=10)\n",
    "\n",
    "                if c in ['ZA', 'TR']:\n",
    "                    axd[iax].text(s=c, x=selfcountry['Year'].values[-1], y=selfcountry['AUC'].values[-1],\n",
    "                            ha='left', va='top', color=color, fontsize=8)\n",
    "                else:\n",
    "                    axd[iax].text(s=c, x=selfcountry['Year'].values[-1], y=selfcountry['AUC'].values[-1],\n",
    "                            ha='left', va='bottom', color=color, fontsize=8)\n",
    "            else:    \n",
    "                axd[iax].plot(selfcountry['Year'].values, selfcountry['AUC'].values, color='0.7', lw=0.1)\n",
    "        #print(c, selfcountry['CitedYear'].values[np.argmin(selfcountry['AUC'].values)], selfcountry['CitedYear'].values[np.argmax(selfcountry['AUC'].values)])\n",
    "\n",
    "axd[\"A\"].set_ylabel('National self-preference', fontsize=lblsize) \n",
    "\n",
    "for iax in [\"B\", \"C\"]:\n",
    "    axd[iax].set_ylim([0.5, 0.7])\n",
    "    #axd[iax].set_yticks([0.5, 0.55, 0.6, 0.65])\n",
    "    axd[iax].set_yticks([0.5, 0.6, 0.7])\n",
    "    axd[iax].set_xlim([1990, 2017]) \n",
    "    axd[iax].set_xticks([1990,2000,2010])\n",
    "\n",
    "    axd[iax].set_ylabel('National self-preference', fontsize=lblsize) \n",
    "    #axd[iax].legend()\n",
    "    \n",
    "    axd[iax].set_xlabel('Year', fontsize=lblsize)\n",
    "    \n",
    "    axd[iax].grid(axis='y', which='major', color='0.8', linestyle='-')\n",
    "    axd[iax].tick_params(axis='y', size=0)\n",
    "    for s in ['top', 'right', 'left']:\n",
    "        axd[iax].spines[s].set_visible(False)\n",
    "\n",
    "#axd[\"D\"].set_ylim([0.5, 0.8])\n",
    "\n",
    "axd[\"B\"].set_xlim([1970, 2017]) \n",
    "axd[\"B\"].set_xticks([1970,1980,1990,2000,2010])\n",
    "\n",
    "mycoefplot(axd[\"D\"], sumdf_topfrac, fontsize=lblsize, ms=50)\n",
    "mycoefplot(axd[\"E\"], sumdf_div, fontsize=lblsize)\n",
    "mycoefplot(axd[\"F\"], sumdf_pat, fontsize=lblsize)\n",
    "\n",
    "\n",
    "for label in [\"A\", \"B\", \"C\"]:\n",
    "    # label physical distance in and down:\n",
    "    trans = mtransforms.ScaledTranslation(-35/72, 7/72, fig.dpi_scale_trans)\n",
    "    axd[label].text(0.0, 1.0, label, transform=axd[label].transAxes + trans,\n",
    "                   fontsize='x-large', va='bottom')#, xx-small, x-small, small, medium, large, x-large, xx-large, larger, smaller\n",
    "\n",
    "for label in [\"D\", \"E\", 'F']:\n",
    "    axd[label].set_aspect(1.75)\n",
    "    trans = mtransforms.ScaledTranslation(-1.59, 7/72, fig.dpi_scale_trans)\n",
    "    axd[label].text(0.0, 1.0, label, transform=axd[label].transAxes + trans,\n",
    "                   fontsize='x-large', va='center')\n",
    "#axd[\"F\"].axis('off')\n",
    "#axd[\"G\"].axis('off')\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "plt.savefig(\"Figures/fig3.pdf\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3df1597",
   "metadata": {},
   "outputs": [],
   "source": [
    "comm2country = {0:['AE', 'BD', 'CN', 'CY', 'DZ', 'EG', 'ID', 'IN', 'IQ', 'IR', 'JO', 'KH', 'KR', 'KW', 'MA', 'MY', 'OM', 'PK', 'QA', 'SA', 'SG', 'TH', 'TN', 'TR', 'TW', 'VN'],\n",
    "1:['AR', 'BN', 'BO', 'BR', 'BW', 'BY', 'CL', 'CM', 'CO', 'CR', 'GE', 'HK', 'HR', 'LU', 'MZ', 'NC', 'PA', 'PE', 'RU', 'SJ', 'ST', 'UY', 'VE'],\n",
    "2:['AT', 'BE', 'CA', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GR', 'HU', 'IE', 'IL', 'IS', 'IT', 'JP', 'MX', 'NL', 'NO', 'NP', 'NZ', 'PH', 'PL', 'PT', 'RO', 'SE', 'SI', 'SK'],\n",
    "3:['AU', 'CH', 'GB', 'US'],\n",
    "4:['BH', 'ET', 'GH', 'KE', 'MW', 'NG', 'TZ', 'UG', 'ZA', 'ZM', 'ZW']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5b8148cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auto\n"
     ]
    }
   ],
   "source": [
    "print(axd[\"D\"].get_aspect())\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "0378b2e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.14285714285714285, 0.7551020408163265)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pyscisci.simpson([0,1,2,3,4,5,6]), pyscisci.simpson([0,0,0,0,0,0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ad285803",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def pvalue2stars(pvalue):\n",
    "    if pvalue < 0.01:\n",
    "        return \"^{***}\"\n",
    "    elif pvalue < 0.05:\n",
    "        return \"^{**}\"\n",
    "    elif pvalue < 0.1:\n",
    "        return \"^{*}\"\n",
    "    else:\n",
    "        return \"\"\n",
    "    \n",
    "def table_row(varname='const', offset=0, roun=2, modellist=[], namedict={}):\n",
    "    row_text = namedict[varname] + \" & \"*(offset+1) + \" & \".join([ \"$\" + str(model.params[varname].round(roun)) + pvalue2stars(model.pvalues[varname]) + \"$\" for model in modellist[offset:]]) + \"\\\\\\\\ \\n\"\n",
    "    row_text +=  \" & \"*(offset+1) + \" & \".join([ \"$(\" + str(model.conf_int().loc[varname]['lower'].round(roun)) + \",\" + str(model.conf_int().loc[varname]['upper'].round(roun)) + \")$\" for model in modellist[offset:]]) + \"\\\\\\\\ \\n\"\n",
    "    row_text +=  \" & \"*(offset+1) + \" & \".join([ \"S.E. $\" + str(model.std_errors[varname].round(roun)) + \"$ p=$\" + str(model.pvalues[varname].round(4)) + \"$\" for model in modellist[offset:]]) + \"\\\\\\\\ [0.8ex]  \\n\"\n",
    "    return row_text\n",
    "\n",
    "def make_latex_table(data, effect_vars = [], exog_var_sets = [], dep_var = \"\", use_const = False, \n",
    "                     normvar = True, namedict = {}, caption_text=\"\"):\n",
    "    \n",
    "    Nmodels = len(exog_var_sets)\n",
    "    \n",
    "    fit_models = []\n",
    "    for icol in range(Nmodels):\n",
    "        exog_vars = list(chain.from_iterable(exog_var_sets[:(icol+1)]))\n",
    "        self_panel = data[effect_vars + exog_vars + [dep_var]].dropna().reset_index(drop=True)\n",
    "        if normvar:\n",
    "            for c in exog_vars + [dep_var]:\n",
    "                self_panel[c] = (self_panel[c] - self_panel[c].mean())/self_panel[c].std()\n",
    "                \n",
    "        self_panel=self_panel.set_index(effect_vars)\n",
    "        if use_const:\n",
    "            exog = sm.add_constant(self_panel[exog_vars])\n",
    "        else:\n",
    "            exog = self_panel[exog_vars]\n",
    "        mod = PanelOLS(self_panel[dep_var], self_panel[exog_vars], entity_effects=True, time_effects=True)\n",
    "        fit_models.append(mod.fit(cov_type='clustered', cluster_entity=True))\n",
    "    \n",
    "    \n",
    "    table_text = \"\"\"\\\\begin{table}[!htb] \\\\centering \n",
    "    \\\\small\n",
    "  \\\\caption{\\\\textbf{National self-preference.}\n",
    "  Model coefficients labeled by $p$-value. Standard errors in parentheses.}  \n",
    "  \\\\label{table:} \n",
    "  \\\\begin{tabular}{l\"\"\" + \"l\"*Nmodels + \"}\"\n",
    "    \n",
    "    table_text += \"\"\"\\\\\\\\[-1.8ex]\\\\hline \n",
    "\\\\hline \\\\\\\\[-1.8ex] \n",
    "\\\\multicolumn{\"\"\" + str(Nmodels+1) + \"\"\"}{c}{\\\\textbf{Dependent variable: Fraction of publications in top venues.}} \\\\\\\\[0.8ex]\\\\hline \n",
    " & \\\\multicolumn{\"\"\" + str(Nmodels) + \"\"\"}{c}{Model}  \\\\\\\\ \"\"\"\n",
    "    \n",
    "    table_text += \"\\cline{2-\" + str(Nmodels + 1) + \"} \\\\\\\\[-1.8ex] \"\n",
    "    \n",
    "    for icol in range(Nmodels):\n",
    "        table_text += \"& (\" + str(icol + 1) + \")\"\n",
    "    \n",
    "    table_text += \" \\\\\\\\ \\hline \\\\\\\\[-1.8ex] \\n\"\n",
    "    \n",
    "    # add constant\n",
    "    if use_const:\n",
    "        table_text += table_row(varname='const', offset=0, roun=2, modellist=fit_models, namedict=namedict)\n",
    "    \n",
    "    for offset, varlist in enumerate(exog_var_sets):\n",
    "        for var in varlist:\n",
    "            table_text += table_row(varname=var, offset=offset, roun=2, modellist=fit_models, namedict=namedict)\n",
    "    \n",
    "    table_text += \"\"\"\\\\hline \n",
    "\\\\hline \\\\\\\\[-1.8ex] \n",
    "\\\\textit{Note:} & \\\\multicolumn{2}{r}{$^{*}p<0.05$; $^{**}p<0.01$; $^{***}p<0.001$} \\\\\\\\ \\n\"\"\"\n",
    "    \n",
    "    table_text += \"Observations & \" + \" & \".join([str(model.nobs) for model in fit_models]) + \" \\\\\\\\ \\n\"\n",
    "    table_text += \"$R^2$ & \" + \" & \".join([str(np.round(model.rsquared, 4)) for model in fit_models]) + \" \\\\\\\\ \\n\"\n",
    "    table_text += \"Log Likelihood & \" + \" & \".join([str(model.loglik.round(2)) for model in fit_models]) + \" \\\\\\\\ \\n\"\n",
    "    table_text += \"F statistic & \" + \" & \".join([\"$\" + str(np.round(model.f_statistic.stat, 2)) + pvalue2stars(model.f_statistic.pval) + \"$ (d.f.=\" + str(model.f_statistic.df) + \")\" for model in fit_models]) + \" \\\\\\\\ \\n\"\n",
    "    table_text += \"\\\\hline \\n\\\\end{tabular} \\n\\\\end{table}\"\n",
    "    \n",
    "    print(table_text)\n",
    "    return table_text\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "09facd6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_net = net_auc[net_auc['CitedYear'] <= 2017].drop_duplicates(subset=['CitingCountry', 'CitedCountry'], keep='last')\n",
    "\n",
    "positive_net['Sign'] = positive_net['Zscore'] > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "8175389b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KernelDensity\n",
    "\n",
    "def make_kde(x, x_d=np.linspace(0, 1, 1000), bandwidth=0.01, color='k', alpha=0.5, ax=None):\n",
    "    kde = KernelDensity(bandwidth=bandwidth, kernel='gaussian')\n",
    "    kde.fit(x[:, None])\n",
    "\n",
    "    # score_samples returns the log of the probability density\n",
    "    logprob = kde.score_samples(x_d[:, None])\n",
    "\n",
    "    ax.fill_between(x_d, np.exp(logprob), alpha=alpha, color=color)\n",
    "    return ax\n",
    "\n",
    "def micro_weak_balance(g):\n",
    "    node_bal = {}\n",
    "    for a in g.nodes():\n",
    "        node_bal[a] = [0,0]\n",
    "        suc = set([s for s in g.successors(a)])\n",
    "        for b in suc:\n",
    "            for c in g.successors(b):\n",
    "                if c in suc:\n",
    "                    if sum([g[a][b]['sign'], g[b][c]['sign'], g[a][c]['sign']]) == 2:\n",
    "                        node_bal[a][1] += 1\n",
    "                    else:\n",
    "                        node_bal[a][0] += 1\n",
    "                        \n",
    "    return node_bal\n",
    "\n",
    "\n",
    "def micro_strong_balance(g):\n",
    "    \n",
    "    good_set = set([0,2])\n",
    "    \n",
    "    node_bal = {}\n",
    "    for a in g.nodes():\n",
    "        node_bal[a] = [0,0]\n",
    "        suc = set([s for s in g.successors(a)])\n",
    "        for b in suc:\n",
    "            for c in g.successors(b):\n",
    "                if c in suc:\n",
    "                    if sum([g[a][b]['sign'], g[b][c]['sign'], g[a][c]['sign']]) in good_set:\n",
    "                        node_bal[a][1] += 1\n",
    "                    else:\n",
    "                        node_bal[a][0] += 1\n",
    "                        \n",
    "    return node_bal\n",
    "  \n",
    "def preserve_sign_rand_net(ndf):\n",
    "    rnet = []\n",
    "    for s in [False, True]:\n",
    "        rsub = ndf[ndf['Sign'] == s][['CitingCountry', 'CitedCountry', 'Sign']].copy()\n",
    "        rsub['CitedCountry'] = rsub['CitedCountry'].sample(frac=1).values \n",
    "        rnet.append(rsub)\n",
    "\n",
    "    rnet = pd.concat(rnet)\n",
    "    return rnet\n",
    "\n",
    "def reverse_neg_edges(ndf):\n",
    "    sub = ndf[ndf['Sign'] == False][['CitingCountry', 'CitedCountry', 'Sign']].copy()\n",
    "    sub['CitedCountry'] = ndf[ndf['Sign'] == False]['CitingCountry'].values\n",
    "    sub['CitingCountry'] = ndf[ndf['Sign'] == False]['CitedCountry'].values\n",
    "    \n",
    "    return pd.concat([ndf[ndf['Sign'] == True][['CitingCountry', 'CitedCountry', 'Sign']].copy(), sub])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "2b7f0a38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32042 [29339  2703]\n",
      "0.9156419699144872\n",
      "0.943179224587388\n"
     ]
    }
   ],
   "source": [
    "nxnet = nx.DiGraph()\n",
    "for s,t,z in positive_net[['CitingCountry', 'CitedCountry', 'Zscore']].values:\n",
    "    nxnet.add_edge(s,t, sign=z>0)\n",
    "node_bal = micro_weak_balance(nxnet)\n",
    "bal = np.array([t[0]/sum(t) for t in node_bal.values() if sum(t) > 0]) \n",
    "\n",
    "tbal = np.array(list(node_bal.values())).sum(axis=0)\n",
    "print(tbal.sum(), tbal)\n",
    "print(tbal[0]/tbal.sum())\n",
    "\n",
    "print(bal.mean())\n",
    "\n",
    "\n",
    "nsamples= 100\n",
    "\n",
    "rweakbal = np.zeros(nsamples)\n",
    "rweakbalnodes = []\n",
    "for isample in range(nsamples):\n",
    "    rnet = preserve_sign_rand_net(positive_net)\n",
    "\n",
    "    rnxnet = nx.DiGraph()\n",
    "    for s,t,z in rnet[['CitingCountry', 'CitedCountry', 'Sign']].values:\n",
    "        rnxnet.add_edge(s,t, sign=z>0)\n",
    "\n",
    "    rnode_bal = micro_weak_balance(rnxnet)\n",
    "    \n",
    "    rweakbalnodes.extend([t[0]/sum(t) for t in rnode_bal.values() if sum(t) > 0]) \n",
    "    \n",
    "    trbal = np.array(list(rnode_bal.values())).sum(axis=0)\n",
    "    \n",
    "    rweakbal[isample] = trbal[0]/trbal.sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "dff4d276",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11989 [9538 2451]\n",
      "0.7955625990491284\n",
      "0.6204220794050506\n"
     ]
    }
   ],
   "source": [
    "print(trbal.sum(), trbal)\n",
    "print(trbal[0]/trbal.sum())\n",
    "\n",
    "#print(trbal.mean())\n",
    "print((bal.mean() - np.mean(rweakbalnodes))/np.std(rweakbalnodes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "b1171b10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 43968 characters]\n",
      "text/plain": [
       "<Figure size 1200x400 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "eurofund_palette = ['#0D95D0', '#7DC462', '#E72F52', '#774FA0', '#EFB743', '#D44627']\n",
    "\n",
    "fig, ax = plt.subplots(1,3,figsize=(12,4))\n",
    "\n",
    "make_kde(bal, x_d=np.linspace(0, 1, 1000), bandwidth=0.02, color=eurofund_palette[0], \n",
    "                 alpha=0.5, ax=ax[0])\n",
    "\n",
    "make_kde(np.array(rweakbalnodes), x_d=np.linspace(0, 1, 1000), bandwidth=0.02, color=eurofund_palette[1], \n",
    "                 alpha=0.5, ax=ax[1])\n",
    "\n",
    "make_kde(rweakbal, x_d=np.linspace(0, 1, 1000), bandwidth=0.01, color=eurofund_palette[3], \n",
    "                 alpha=0.5, ax=ax[2])\n",
    "\n",
    "\n",
    "for iax in range(3):\n",
    "    ax[iax].tick_params(axis='x', labelsize=12)\n",
    "    ax[iax].tick_params(axis='y', labelsize=12)\n",
    "    ax[iax].set_ylabel('Density', fontsize=12)\n",
    "    ax[iax].set_xlabel('Node balance', fontsize=12)\n",
    "    \n",
    "    \n",
    "for iax,label in enumerate([\"A\", \"B\", \"C\"]):\n",
    "    # label physical distance in and down:\n",
    "    trans = mtransforms.ScaledTranslation(-40/72, 0/72, fig.dpi_scale_trans)\n",
    "    ax[iax].text(0.0, 1.0, label, transform=ax[iax].transAxes + trans,\n",
    "                   fontsize='xx-large', va='bottom')#, xx-small, x-small, small, medium, large, x-large, xx-large, larger, smaller\n",
    "\n",
    "plt.tight_layout()    \n",
    "#ax.set_xlabel('Node balance', fontsize=20)\n",
    "plt.savefig('Figures/weakbalance.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7667da1b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
