{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "874e645e",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "\n",
    "import pyscisci.all as pyscisci\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "import nltk\n",
    "#nltk.download('punkt')\n",
    "import string\n",
    "import re\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "122e3e61",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "# set this path to where the OpenAlex database will be stored\n",
    "path2openalex = '/Users/hgt6rn/Documents/DataSets/OpenAlex'\n",
    "\n",
    "path2countrydata = \"/Users/hgt6rn/Documents/DataSets/OpenAlex/precomputed_metrics\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "61b2080c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "myoa = pyscisci.OpenAlex(path2openalex, database_extension='csv.gz', keep_in_memory=False) \n",
    "# set keep_in_memory=False if you want to load the database each time its needed - good for when you \n",
    "# cant keep more than one dataframe in memory at a time\n",
    "# otherwise keep_in_memory=True will keep each database in memory after its loaded"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17dd8300",
   "metadata": {},
   "source": [
    "# Create the publication to country mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "76795afc",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98e68edc9e174195ba3fd17d384bfc90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publications:   0%|          | 0/513 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (9,14) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (9) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (9,10) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(244557590, 4)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub = myoa.load_publications(columns =['PublicationId', 'Year', 'DocType', 'JournalId'])\n",
    "pub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "69cff5cc",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "journal-article        123396340\n",
       "book-chapter            19535623\n",
       "proceedings-article      8897534\n",
       "dissertation             6462566\n",
       "book                     4711652\n",
       "posted-content           4699849\n",
       "dataset                  2854817\n",
       "reference-entry          1096008\n",
       "journal-issue             979854\n",
       "other                     817895\n",
       "report                    776182\n",
       "monograph                 570396\n",
       "standard                  361932\n",
       "peer-review               302961\n",
       "reference-book            222188\n",
       "component                 202203\n",
       "proceedings               153415\n",
       "journal                    55161\n",
       "grant                      28321\n",
       "report-series              18541\n",
       "book-part                  16214\n",
       "journal-volume              7902\n",
       "book-series                 3144\n",
       "proceedings-series          1688\n",
       "book-set                     436\n",
       "Name: DocType, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub['DocType'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "03afa548",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(153910134, 4)\n",
      "(129520147, 4)\n",
      "(128228946, 4)\n"
     ]
    }
   ],
   "source": [
    "kept_doctypes = np.sort(['journal-article', 'proceedings-article'])\n",
    "pub.dropna(inplace=True)\n",
    "print(pub.shape)\n",
    "pub = pub[pyscisci.isin_sorted(pub['DocType'].values, kept_doctypes)].reset_index(drop=True)\n",
    "print(pub.shape)\n",
    "pub = pub[pub['Year'] >= 1900].reset_index(drop=True)\n",
    "print(pub.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3f5acc0f",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2year = {pid:y for pid,y in pub[['PublicationId', 'Year']].values}\n",
    "focus_pubs = np.sort(pub['PublicationId'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "64777454",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "del pub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cd0f9a14",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbd7ed3624bb4b098f0740f69fbdfa3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Affiliations:   0%|          | 0/153 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AffiliationId</th>\n",
       "      <th>Country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4210161801</td>\n",
       "      <td>GB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4210127386</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4210157748</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4210095232</td>\n",
       "      <td>IL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4210128266</td>\n",
       "      <td>BE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107667</th>\n",
       "      <td>4210124170</td>\n",
       "      <td>DK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107668</th>\n",
       "      <td>4210119575</td>\n",
       "      <td>ES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107669</th>\n",
       "      <td>4210158772</td>\n",
       "      <td>FI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107670</th>\n",
       "      <td>4210135183</td>\n",
       "      <td>CN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107671</th>\n",
       "      <td>4210131651</td>\n",
       "      <td>BR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>107672 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        AffiliationId Country\n",
       "0          4210161801      GB\n",
       "1          4210127386      US\n",
       "2          4210157748      US\n",
       "3          4210095232      IL\n",
       "4          4210128266      BE\n",
       "...               ...     ...\n",
       "107667     4210124170      DK\n",
       "107668     4210119575      ES\n",
       "107669     4210158772      FI\n",
       "107670     4210135183      CN\n",
       "107671     4210131651      BR\n",
       "\n",
       "[107672 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aff_df = myoa.load_affiliations(columns = ['AffiliationId', 'Country'])\n",
    "aff_df.dropna(inplace=True)\n",
    "aff_df.reset_index(drop=True, inplace=True)\n",
    "aff_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6b1d4ab",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "\n",
    "focus_affs = np.sort(aff_df['AffiliationId'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "648e101d",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16dbb3ffd0a34e81ba92cdbafb892096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publication Author Affiliation:   0%|          | 0/513 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(227796792, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PublicationId    63983125\n",
       "AuthorId         60791118\n",
       "AffiliationId       94432\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paa_df = myoa.load_publicationauthoraffiliation(columns=['PublicationId', 'AuthorId', 'AffiliationId'],\n",
    "                                                dropna=['AuthorId', 'AffiliationId'],\n",
    "                                                filter_dict = {'PublicationId':focus_pubs,\n",
    "                                                             'AffiliationId':focus_affs})\n",
    "print(paa_df.shape)\n",
    "paa_df.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7057ae44",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(227796792, 4)\n"
     ]
    }
   ],
   "source": [
    "paa_df = paa_df.merge(aff_df, how='left', on='AffiliationId')\n",
    "paa_df.dropna(inplace=True)\n",
    "print(paa_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "676bccc1",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "del aff_df\n",
    "del focus_pubs\n",
    "del focus_affs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b429cbd9",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PublicationId    63983125\n",
       "Country               223\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub2country = paa_df[['PublicationId', 'Country']].drop_duplicates(subset=['PublicationId', 'Country'])\n",
    "pub2country.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bad485b3",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2country.to_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"), compression='gzip', mode='w', \n",
    "                   header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "087aef3e",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub[['PublicationId', 'Year']].dropna().astype(int).to_csv(os.path.join(path2countrydata, \"oa_pub2year.csv.gz\"), compression='gzip', mode='w', \n",
    "                   header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fa592cfb",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>NumberPublications</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AD</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AE</td>\n",
       "      <td>81660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AF</td>\n",
       "      <td>2572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AG</td>\n",
       "      <td>4547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AL</td>\n",
       "      <td>9224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>XK</td>\n",
       "      <td>6113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>YE</td>\n",
       "      <td>8512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>ZA</td>\n",
       "      <td>426873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>ZM</td>\n",
       "      <td>11393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>ZW</td>\n",
       "      <td>18436</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>223 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Country  NumberPublications\n",
       "0        AD                 131\n",
       "1        AE               81660\n",
       "2        AF                2572\n",
       "3        AG                4547\n",
       "4        AL                9224\n",
       "..      ...                 ...\n",
       "218      XK                6113\n",
       "219      YE                8512\n",
       "220      ZA              426873\n",
       "221      ZM               11393\n",
       "222      ZW               18436\n",
       "\n",
       "[223 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "npubs = pub2country.groupby('Country')['PublicationId'].nunique().to_frame().reset_index()\n",
    "npubs = npubs.rename(columns={'PublicationId':'NumberPublications'})\n",
    "\n",
    "npubs.to_csv(os.path.join(path2countrydata, 'oa_countryproductivity.csv.gz'), \n",
    "                          compression='gzip', mode='w', header=True, index=False)\n",
    "\n",
    "npubs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "df60fa46",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Year</th>\n",
       "      <th>NumberPublications</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AD</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AD</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AD</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AD</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AD</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16964</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16965</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>1449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16966</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>1750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16967</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>1307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16968</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2023.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16969 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Country    Year  NumberPublications\n",
       "0          AD  1990.0                   1\n",
       "1          AD  1993.0                   2\n",
       "2          AD  1998.0                   1\n",
       "3          AD  2003.0                   1\n",
       "4          AD  2006.0                   4\n",
       "...       ...     ...                 ...\n",
       "16964      ZW  2019.0                1125\n",
       "16965      ZW  2020.0                1449\n",
       "16966      ZW  2021.0                1750\n",
       "16967      ZW  2022.0                1307\n",
       "16968      ZW  2023.0                   4\n",
       "\n",
       "[16969 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub2country = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"))\n",
    "                   \n",
    "pub2country['Year'] = pub2country['PublicationId'].map(pub2year)\n",
    "\n",
    "npubs_y = pub2country.groupby(['Country', 'Year'])['PublicationId'].nunique().to_frame().reset_index()\n",
    "npubs_y = npubs_y.rename(columns={'PublicationId':'NumberPublications'})\n",
    "\n",
    "npubs_y.to_csv(os.path.join(path2countrydata, 'oa_countryproductivity_yearly.csv.gz'), \n",
    "                          compression='gzip', mode='w', header=True, index=False)\n",
    "\n",
    "npubs_y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93fd610d",
   "metadata": {},
   "source": [
    "# Create the yearly national citation distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5522fc3b",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(63983125,)\n"
     ]
    }
   ],
   "source": [
    "pub2country = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"))\n",
    "\n",
    "kept_pids = np.sort(pub2country['PublicationId'].unique())\n",
    "#kept_pids = np.sort(paa_df['PublicationId'].unique())\n",
    "print(kept_pids.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2b568a59",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2year = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2year.csv.gz\"))\n",
    "#pub2year = {pid:y for pid,y in pub2year[['PublicationId', 'Year']].values}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f8daa526",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PublicationId    57558268\n",
       "Country               223\n",
       "Year                   34\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#pub2country = pub2country.merge(pub2year, how='left', on='PublicationId')\n",
    "pub2country[np.logical_and(pub2country['Year'] >= 1990, pub2country['Year'] <= 2023)].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8ece6c21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(57558268,)\n"
     ]
    }
   ],
   "source": [
    "# change the timeperiod to focus on\n",
    "\n",
    "window_start = 1990\n",
    "window_end = 2023\n",
    "\n",
    "kept_pids = np.sort(pub2country[np.logical_and(pub2country['Year'] >= window_start, pub2country['Year'] <= window_end)]['PublicationId'].unique())\n",
    "print(kept_pids.shape)\n",
    "pub2year = {pid:y for pid,y in pub2year[['PublicationId', 'Year']].values}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fe01c9c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "# Here we use a citation window\n",
    "citation_window = 5\n",
    "\n",
    "pub2ref = []\n",
    "for fname in os.listdir(os.path.join(path2openalex, 'pub2ref')):\n",
    "    if '.csv.gz' in fname:\n",
    "        df = pd.read_csv(os.path.join(path2openalex, 'pub2ref', fname))\n",
    "        \n",
    "        df = df[pyscisci.isin_sorted(df['CitingPublicationId'].values, kept_pids)]\n",
    "        df = df[pyscisci.isin_sorted(df['CitedPublicationId'].values, kept_pids)]\n",
    "        \n",
    "        df['CitingYear'] = [pub2year.get(pid, None) for pid in df['CitingPublicationId'].values]\n",
    "        df.dropna(inplace=True)\n",
    "        \n",
    "        df['CitedYear'] = [pub2year.get(pid, None) for pid in df['CitedPublicationId'].values]\n",
    "        df.dropna(inplace=True)\n",
    "        \n",
    "        df = df.loc[df['CitingYear'] >= df['CitedYear']]\n",
    "        df = df.loc[df['CitingYear'] <=  df['CitedYear'].values + citation_window]\n",
    "        \n",
    "        pub2ref.append(df[['CitingPublicationId', 'CitingYear', 'CitedPublicationId', 'CitedYear']])\n",
    "pub2ref = pd.concat(pub2ref)\n",
    "print(pub2ref.shape)\n",
    "pub2ref.drop_duplicates(subset=['CitingPublicationId', 'CitedPublicationId'], inplace=True)\n",
    "print(pub2ref.shape)\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5707178e",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CitingPublicationId    41431724\n",
       "CitedPublicationId     30556883\n",
       "CitedYear                   118\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change the 2017 if you have a more recent snapshot of OpenAlex\n",
    "\n",
    "pub2ref = pub2ref[pub2ref['CitedYear'] <= 2017].reset_index(drop=True)\n",
    "del pub2ref['CitingYear']\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13abcee1",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2ref = pub2ref.merge(pub2country[['PublicationId', 'Country']], how='left', left_on = 'CitingPublicationId', right_on='PublicationId')\n",
    "del pub2ref['PublicationId']\n",
    "pub2ref.rename(columns={'Country':'CitingCountry'}, inplace=True)\n",
    "\n",
    "pub2ref = pub2ref.merge(pub2country[['PublicationId', 'Country']], how='left', left_on = 'CitedPublicationId', right_on='PublicationId')\n",
    "del pub2ref['PublicationId']\n",
    "pub2ref.rename(columns={'Country':'CitedCountry'}, inplace=True)\n",
    "\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e67afce0",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e9b4dd75",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "ncites = pub2ref.groupby('CitedPublicationId', as_index=False)['CitingPublicationId'].nunique()\n",
    "ncites = ncites.rename(columns={'CitingPublicationId':'C5'})\n",
    "ncites = ncites.merge(pub2country[['PublicationId', 'Country']], how='left', left_on = 'CitedPublicationId', right_on='PublicationId')\n",
    "del ncites['PublicationId']\n",
    "ncites['CitedYear'] = ncites['CitedPublicationId'].map(pub2year)\n",
    "ncites.rename(columns={'Country':'CitedCountry'}, inplace=True)\n",
    "\n",
    "ncites.to_csv(os.path.join(path2countrydata, \"oa_c5.csv.gz\"), compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3dccbc48",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(242110245, 5)\n"
     ]
    }
   ],
   "source": [
    "countrycites = pub2ref.groupby(['CitedPublicationId', 'CitingCountry', 'CitedYear'], as_index=False)['CitingPublicationId'].nunique()\n",
    "\n",
    "countrycites = countrycites.merge(pub2country, how='left', left_on='CitedPublicationId', right_on='PublicationId')\n",
    "del countrycites['PublicationId']\n",
    "countrycites = countrycites.rename(columns={'CitingPublicationId':'CountryCitations'})\n",
    "print(countrycites.shape)\n",
    "countrycites.to_csv(os.path.join(path2countrydata, \"oa_countrycites.csv.gz\"), compression='gzip', mode='w', header=True, index=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55b93f11",
   "metadata": {},
   "source": [
    "# remove self-citations and same-organization citations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cc82fcb5",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65018678 65018678\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pub2authors = defaultdict(set)\n",
    "pub2aff = defaultdict(set)\n",
    "for pid, aid, aff in paa_df[['PublicationId', 'AuthorId', 'AffiliationId']].values:\n",
    "    pub2authors[pid].add(aid)\n",
    "    pub2aff[pid].add(aff)\n",
    "    \n",
    "print(len(pub2authors), len(pub2aff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dc782769",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "del paa_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03c9367c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "noselfcite = np.array([len(pub2authors[citingpid] & pub2authors[citedpid]) == 0 for citingpid, citedpid in pub2ref[['CitingPublicationId', 'CitedPublicationId']].values])\n",
    "print(np.count_nonzero(noselfcite==0))\n",
    "\n",
    "\n",
    "countrycites = pub2ref.loc[noselfcite].groupby(['CitedPublicationId', 'CitingCountry', 'CitedYear'], as_index=False)['CitingPublicationId'].nunique()\n",
    "countrycites = countrycites.merge(pub2country, how='left', left_on='CitedPublicationId', right_on='PublicationId')\n",
    "del countrycites['PublicationId']\n",
    "countrycites = countrycites.rename(columns={'CitingPublicationId':'CountryCitations'})\n",
    "\n",
    "countrycites.to_csv(os.path.join(path2countrydata, \"oa_countrycites_noself.csv.gz\"), compression='gzip', mode='w', header=True, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cef6f847",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100036381\n",
      "111793438\n"
     ]
    }
   ],
   "source": [
    "notsameorg = np.array([len(pub2aff[citingpid] & pub2aff[citedpid]) == 0 for citingpid, citedpid in pub2ref[['CitingPublicationId', 'CitedPublicationId']].values])\n",
    "print(np.count_nonzero(notsameorg==0))\n",
    "notsameorg = np.logical_and(notsameorg, noselfcite)\n",
    "print(np.count_nonzero(notsameorg==0))\n",
    "\n",
    "countrycites = pub2ref.loc[notsameorg].groupby(['CitedPublicationId', 'CitingCountry', 'CitedYear'], as_index=False)['CitingPublicationId'].nunique()\n",
    "countrycites = countrycites.merge(pub2country, how='left', left_on='CitedPublicationId', right_on='PublicationId')\n",
    "del countrycites['PublicationId']\n",
    "countrycites = countrycites.rename(columns={'CitingPublicationId':'CountryCitations'})\n",
    "\n",
    "countrycites.to_csv(os.path.join(path2countrydata, \"oa_countrycites_nosameorg.csv.gz\"), compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31076d1d",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# now get the collaboration networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "473e8759",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2country = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "67ad264c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2country['Year'] = pub2country['PublicationId'].map(pub2year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5be6c056",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Country</th>\n",
       "      <th>nCountries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.147478e+09</td>\n",
       "      <td>BR</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.205869e+09</td>\n",
       "      <td>FR</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.225966e+09</td>\n",
       "      <td>BE</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.471332e+09</td>\n",
       "      <td>CN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.367226e+09</td>\n",
       "      <td>CN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76593135</th>\n",
       "      <td>3.022242e+09</td>\n",
       "      <td>GB</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76593136</th>\n",
       "      <td>2.396241e+09</td>\n",
       "      <td>CN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76593137</th>\n",
       "      <td>2.358301e+09</td>\n",
       "      <td>CN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76593138</th>\n",
       "      <td>2.953066e+09</td>\n",
       "      <td>US</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76593139</th>\n",
       "      <td>2.166353e+09</td>\n",
       "      <td>TH</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>76593140 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          PublicationId Country  nCountries\n",
       "0          3.147478e+09      BR           1\n",
       "1          4.205869e+09      FR           1\n",
       "2          4.225966e+09      BE           1\n",
       "3          2.471332e+09      CN           1\n",
       "4          2.367226e+09      CN           1\n",
       "...                 ...     ...         ...\n",
       "76593135   3.022242e+09      GB           1\n",
       "76593136   2.396241e+09      CN           1\n",
       "76593137   2.358301e+09      CN           1\n",
       "76593138   2.953066e+09      US           1\n",
       "76593139   2.166353e+09      TH           1\n",
       "\n",
       "[76593140 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Limited perspectives: The group may become less diverse in terms of expertise, \n",
    "# which could limit the range of perspectives and ideas brought to bear on scientific problems. This could result in a narrower understanding of issues, and may make it more difficult to identify and solve problems.\n",
    "\n",
    "#Reduced innovation: When there is a lack of diversity in a group of scientists, there is also a risk of stagnation and reduced innovation. Without new and varied ideas, the group may struggle to come up with creative solutions to problems or develop new approaches to research.\n",
    "\n",
    "#Less collaboration: As the group becomes more insular, they may become less open to collaboration with researchers outside their immediate circle. This can limit opportunities for interdisciplinary work and the sharing of knowledge across different fields, which could ultimately slow down scientific progress.\n",
    "\n",
    "pub2country['nCountries'] = pub2country.groupby(['PublicationId'])['Country'].transform('nunique')\n",
    "pub2country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e8602f31",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2country['IsNational'] = pub2country['nCountries'] == 1\n",
    "#national_rate = pub2country.groupby(['Country', 'Year'], as_index=False)['IsNational'].apply(lambda x: x.values.sum()/x.shape[0])\n",
    "#national_rate.rename(columns = {'IsNational':'FractionNationalAuthors'}, inplace=True)\n",
    "\n",
    "#national_rate.to_csv(os.path.join(path2countrydata, \"oa_national_authorship_rate.csv.gz\"), \n",
    "#                                 compression='gzip', mode='w', header=True, index=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c4cf5581",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "cocountry = pub2country[~pub2country['IsNational']].reset_index(drop=True)\n",
    "country2int = {c:i for i, c in enumerate(np.sort(cocountry['Country'].unique()))}\n",
    "with open(os.path.join(path2countrydata, \"oa_cocountry_collaboration_network_country2int.json\"), 'w') as outfile:\n",
    "    outfile.write(json.dumps(country2int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6edbfad1",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1970\n",
      "1971\n",
      "1972\n",
      "1973\n",
      "1974\n",
      "1975\n",
      "1976\n",
      "1977\n",
      "1978\n",
      "1979\n",
      "1980\n",
      "1981\n",
      "1982\n",
      "1983\n",
      "1984\n",
      "1985\n",
      "1986\n",
      "1987\n",
      "1988\n",
      "1989\n",
      "1990\n",
      "1991\n",
      "1992\n",
      "1993\n",
      "1994\n",
      "1995\n",
      "1996\n",
      "1997\n",
      "1998\n",
      "1999\n",
      "2000\n",
      "2001\n",
      "2002\n",
      "2003\n",
      "2004\n",
      "2005\n",
      "2006\n",
      "2007\n",
      "2008\n",
      "2009\n",
      "2010\n",
      "2011\n",
      "2012\n",
      "2013\n",
      "2014\n",
      "2015\n",
      "2016\n",
      "2017\n",
      "2018\n",
      "2019\n",
      "2020\n",
      "2021\n",
      "2022\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "_save_dispatcher() missing 1 required positional argument: 'arr'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [35], line 29\u001b[0m\n\u001b[1;32m     26\u001b[0m     cocountry_net[y\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1970\u001b[39m] \u001b[38;5;241m=\u001b[39m adj_mat\u001b[38;5;241m.\u001b[39mtodense()\n\u001b[1;32m     27\u001b[0m     \u001b[38;5;28mprint\u001b[39m(y)\n\u001b[0;32m---> 29\u001b[0m \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave\u001b[49m\u001b[43m(\u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath2countrydata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43moa_cocountry_collaboration_network.npy\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m<__array_function__ internals>:179\u001b[0m, in \u001b[0;36msave\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: _save_dispatcher() missing 1 required positional argument: 'arr'"
     ]
    }
   ],
   "source": [
    "cocountry = pub2country[~pub2country['IsNational']].reset_index(drop=True)\n",
    "#years = np.sort(cocountry['Year'].unique())\n",
    "years = np.arange(1970, 2023)\n",
    "Nyears = years.shape[0]\n",
    "\n",
    "country2int = {c:i for i, c in enumerate(np.sort(cocountry['Country'].unique()))}\n",
    "int2country = {i:c for i, c in country2int.items()}\n",
    "\n",
    "cocountry['Country'] = cocountry['Country'].map(country2int)\n",
    "Ncountry = cocountry['Country'].nunique()\n",
    "\n",
    "pub2int = {pid:i for i, pid in enumerate(np.sort(cocountry['PublicationId'].unique()))}\n",
    "Npubs = cocountry['PublicationId'].nunique()\n",
    "cocountry['PublicationId'] = cocountry['PublicationId'].map(country2int)\n",
    "\n",
    "cocountry_net = np.zeros(shape=(Nyears,Ncountry,Ncountry))\n",
    "for y in years:\n",
    "    bipartite_adj = pyscisci.dataframe2bipartite(cocountry.loc[cocountry['Year'] == y], \n",
    "                                        'Country', 'PublicationId', (Ncountry, Npubs) )\n",
    "    \n",
    "    adj_mat = pyscisci.project_bipartite_mat(bipartite_adj, project_to = 'row')\n",
    "\n",
    "    adj_mat.setdiag(0)\n",
    "    adj_mat.eliminate_zeros()\n",
    "    \n",
    "    cocountry_net[y-1970] = adj_mat.todense()\n",
    "    print(y)\n",
    "    \n",
    "np.save(os.path.join(path2countrydata, \"oa_cocountry_collaboration_network.npy\"), arr = cocountry_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0718f816",
   "metadata": {},
   "source": [
    "# Diversity of countries publishing in journals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e5fc61c2",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Year</th>\n",
       "      <th>DocType</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>Country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3147477878</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>4.210191e+09</td>\n",
       "      <td>BR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3092696942</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>9.868503e+07</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2996677922</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>4.210208e+09</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3128509038</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>4.210212e+09</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4205868985</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>proceedings-article</td>\n",
       "      <td>1.062967e+08</td>\n",
       "      <td>FR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140838956</th>\n",
       "      <td>2358301002</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>2.764507e+09</td>\n",
       "      <td>CN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140838957</th>\n",
       "      <td>2953065950</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>4.210235e+09</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140838958</th>\n",
       "      <td>1799263988</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>2.764706e+09</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140838959</th>\n",
       "      <td>2166352754</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>2.751447e+09</td>\n",
       "      <td>TH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140838960</th>\n",
       "      <td>1741139020</td>\n",
       "      <td>2015.0</td>\n",
       "      <td>journal-article</td>\n",
       "      <td>2.596512e+09</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>140838961 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PublicationId    Year              DocType     JournalId Country\n",
       "0             3147477878  2021.0      journal-article  4.210191e+09      BR\n",
       "1             3092696942  2020.0      journal-article  9.868503e+07     NaN\n",
       "2             2996677922  2019.0      journal-article  4.210208e+09     NaN\n",
       "3             3128509038  2019.0      journal-article  4.210212e+09     NaN\n",
       "4             4205868985  2019.0  proceedings-article  1.062967e+08      FR\n",
       "...                  ...     ...                  ...           ...     ...\n",
       "140838956     2358301002  2009.0      journal-article  2.764507e+09      CN\n",
       "140838957     2953065950  2020.0      journal-article  4.210235e+09      US\n",
       "140838958     1799263988  2013.0      journal-article  2.764706e+09     NaN\n",
       "140838959     2166352754  2006.0      journal-article  2.751447e+09      TH\n",
       "140838960     1741139020  2015.0      journal-article  2.596512e+09     NaN\n",
       "\n",
       "[140838961 rows x 5 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub = pub.merge(pub2country, how='left', on='PublicationId')\n",
    "pub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "63e14d2f",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>JournalId</th>\n",
       "      <th>Year</th>\n",
       "      <th>CountryDiversity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.166100e+04</td>\n",
       "      <td>1992.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.166100e+04</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>0.689877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.166100e+04</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>0.751634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.166100e+04</td>\n",
       "      <td>1995.0</td>\n",
       "      <td>0.611852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.166100e+04</td>\n",
       "      <td>1996.0</td>\n",
       "      <td>0.532407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062569</th>\n",
       "      <td>4.306541e+09</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062570</th>\n",
       "      <td>4.306549e+09</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062571</th>\n",
       "      <td>4.306551e+09</td>\n",
       "      <td>1994.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062572</th>\n",
       "      <td>4.306555e+09</td>\n",
       "      <td>1975.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062573</th>\n",
       "      <td>4.306557e+09</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1062574 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            JournalId    Year  CountryDiversity\n",
       "0        6.166100e+04  1992.0          1.000000\n",
       "1        6.166100e+04  1993.0          0.689877\n",
       "2        6.166100e+04  1994.0          0.751634\n",
       "3        6.166100e+04  1995.0          0.611852\n",
       "4        6.166100e+04  1996.0          0.532407\n",
       "...               ...     ...               ...\n",
       "1062569  4.306541e+09  2016.0          1.000000\n",
       "1062570  4.306549e+09  2001.0          1.000000\n",
       "1062571  4.306551e+09  1994.0          1.000000\n",
       "1062572  4.306555e+09  1975.0          1.000000\n",
       "1062573  4.306557e+09  2008.0          0.500000\n",
       "\n",
       "[1062574 rows x 3 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countrydiversity = pub.dropna(subset=['JournalId', 'Country', 'Year']).groupby(['JournalId', 'Year'], as_index=False)['Country'].apply(pyscisci.simpson)\n",
    "countrydiversity.rename(columns={'Country':'CountryDiversity'}, inplace=True)\n",
    "countrydiversity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "031c41da",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "countrydiversity.to_csv(os.path.join(path2countrydata, \"oa_journal_diversity.csv.gz\"), \n",
    "                                 compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "683a6483",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Year</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>CountryDiversity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AD</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>1.043534e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>0.329370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AD</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>5.287739e+07</td>\n",
       "      <td>1</td>\n",
       "      <td>0.079967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AD</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>9.208579e+07</td>\n",
       "      <td>1</td>\n",
       "      <td>0.268879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AD</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>4.306421e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>0.103190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AD</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>9.906897e+07</td>\n",
       "      <td>1</td>\n",
       "      <td>0.609204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203976</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>4.306403e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>0.398645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203977</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2023.0</td>\n",
       "      <td>8.501616e+07</td>\n",
       "      <td>1</td>\n",
       "      <td>0.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203978</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2023.0</td>\n",
       "      <td>1.852973e+08</td>\n",
       "      <td>1</td>\n",
       "      <td>0.189014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203979</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2023.0</td>\n",
       "      <td>4.210202e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>0.283951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203980</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2023.0</td>\n",
       "      <td>4.210205e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>0.206400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10203981 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Country    Year     JournalId  PublicationId  CountryDiversity\n",
       "0             AD  1990.0  1.043534e+09              1          0.329370\n",
       "1             AD  1993.0  5.287739e+07              1          0.079967\n",
       "2             AD  1993.0  9.208579e+07              1          0.268879\n",
       "3             AD  1998.0  4.306421e+09              1          0.103190\n",
       "4             AD  2003.0  9.906897e+07              1          0.609204\n",
       "...          ...     ...           ...            ...               ...\n",
       "10203976      ZW  2022.0  4.306403e+09              1          0.398645\n",
       "10203977      ZW  2023.0  8.501616e+07              1          0.375000\n",
       "10203978      ZW  2023.0  1.852973e+08              1          0.189014\n",
       "10203979      ZW  2023.0  4.210202e+09              1          0.283951\n",
       "10203980      ZW  2023.0  4.210205e+09              1          0.206400\n",
       "\n",
       "[10203981 rows x 5 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubweight = pub.groupby(['Country', 'Year', 'JournalId'], as_index=False)['PublicationId'].nunique()\n",
    "pubweight = pubweight.merge(countrydiversity, how='left', on=['JournalId', 'Year'])\n",
    "pubweight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f9fc721d",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Year</th>\n",
       "      <th>AveJournalDiversity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AD</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>0.329370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AD</td>\n",
       "      <td>1993.0</td>\n",
       "      <td>0.174423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AD</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>0.103190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AD</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>0.609204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AD</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>0.468153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16949</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.193901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16950</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>0.218544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16951</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>0.203225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16952</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>0.200901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16953</th>\n",
       "      <td>ZW</td>\n",
       "      <td>2023.0</td>\n",
       "      <td>0.263591</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16954 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Country    Year  AveJournalDiversity\n",
       "0          AD  1990.0             0.329370\n",
       "1          AD  1993.0             0.174423\n",
       "2          AD  1998.0             0.103190\n",
       "3          AD  2003.0             0.609204\n",
       "4          AD  2006.0             0.468153\n",
       "...       ...     ...                  ...\n",
       "16949      ZW  2019.0             0.193901\n",
       "16950      ZW  2020.0             0.218544\n",
       "16951      ZW  2021.0             0.203225\n",
       "16952      ZW  2022.0             0.200901\n",
       "16953      ZW  2023.0             0.263591\n",
       "\n",
       "[16954 rows x 3 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def weighted_sum(x):\n",
    "    return np.dot(x['PublicationId'].values, x['CountryDiversity'].values)/x['PublicationId'].sum()\n",
    "\n",
    "country_journal_diversity = pubweight.groupby(['Country', 'Year'], as_index=False).apply(weighted_sum)\n",
    "country_journal_diversity.rename(columns={None:'AveJournalDiversity'}, inplace=True)\n",
    "country_journal_diversity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8181f36c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "country_journal_diversity.to_csv(os.path.join(path2countrydata, \"oa_country_journal_diversity.csv.gz\"), \n",
    "                                 compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "12d8a48c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3333333333333333"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pyscisci.simpson(['a', 'b', 'c'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25aa4025",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Topic Diversity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "953c2dec",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pub2country = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1347ad3c",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43445e27e35d4f2f80a32d38a0423dad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Fields:   0%|          | 0/513 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(877352959, 2)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "focus_pubs = np.sort(pub2country['PublicationId'].unique())\n",
    "pub2field = myoa.load_pub2field(columns=['PublicationId', 'FieldId'], filter_dict={'PublicationId':focus_pubs})\n",
    "pub2field.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2787bd4a",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FieldId</th>\n",
       "      <th>FieldName</th>\n",
       "      <th>WikiData</th>\n",
       "      <th>FieldLevel</th>\n",
       "      <th>NumberPublications</th>\n",
       "      <th>NumberCitations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>150631331</td>\n",
       "      <td>Shell script</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q959549</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>196351127</td>\n",
       "      <td>Strategic defence</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q7621831</td>\n",
       "      <td>3</td>\n",
       "      <td>522</td>\n",
       "      <td>747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28838541</td>\n",
       "      <td>Flanging</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q1323480</td>\n",
       "      <td>2</td>\n",
       "      <td>864</td>\n",
       "      <td>3198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2778416019</td>\n",
       "      <td>Natural person</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q154954</td>\n",
       "      <td>2</td>\n",
       "      <td>664</td>\n",
       "      <td>230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>206613426</td>\n",
       "      <td>Traditional economy</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q2994466</td>\n",
       "      <td>2</td>\n",
       "      <td>570</td>\n",
       "      <td>871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65068</th>\n",
       "      <td>163439950</td>\n",
       "      <td>Data</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q494756</td>\n",
       "      <td>2</td>\n",
       "      <td>86</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65069</th>\n",
       "      <td>8324905</td>\n",
       "      <td>Uniform Code of Military Justice</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q1424958</td>\n",
       "      <td>5</td>\n",
       "      <td>149</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65070</th>\n",
       "      <td>9335704</td>\n",
       "      <td>AP United States Government and Politics</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q4653504</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65071</th>\n",
       "      <td>70784383</td>\n",
       "      <td>Nuclear reactor safety systems</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q521918</td>\n",
       "      <td>3</td>\n",
       "      <td>118</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65072</th>\n",
       "      <td>64277026</td>\n",
       "      <td>Italian literature</td>\n",
       "      <td>https://www.wikidata.org/wiki/Q74217</td>\n",
       "      <td>2</td>\n",
       "      <td>37</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65073 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          FieldId                                 FieldName  \\\n",
       "0       150631331                              Shell script   \n",
       "1       196351127                         Strategic defence   \n",
       "2        28838541                                  Flanging   \n",
       "3      2778416019                            Natural person   \n",
       "4       206613426                       Traditional economy   \n",
       "...           ...                                       ...   \n",
       "65068   163439950                                      Data   \n",
       "65069     8324905          Uniform Code of Military Justice   \n",
       "65070     9335704  AP United States Government and Politics   \n",
       "65071    70784383            Nuclear reactor safety systems   \n",
       "65072    64277026                        Italian literature   \n",
       "\n",
       "                                     WikiData  FieldLevel  NumberPublications  \\\n",
       "0       https://www.wikidata.org/wiki/Q959549           4                   0   \n",
       "1      https://www.wikidata.org/wiki/Q7621831           3                 522   \n",
       "2      https://www.wikidata.org/wiki/Q1323480           2                 864   \n",
       "3       https://www.wikidata.org/wiki/Q154954           2                 664   \n",
       "4      https://www.wikidata.org/wiki/Q2994466           2                 570   \n",
       "...                                       ...         ...                 ...   \n",
       "65068   https://www.wikidata.org/wiki/Q494756           2                  86   \n",
       "65069  https://www.wikidata.org/wiki/Q1424958           5                 149   \n",
       "65070  https://www.wikidata.org/wiki/Q4653504           4                  40   \n",
       "65071   https://www.wikidata.org/wiki/Q521918           3                 118   \n",
       "65072    https://www.wikidata.org/wiki/Q74217           2                  37   \n",
       "\n",
       "       NumberCitations  \n",
       "0                    0  \n",
       "1                  747  \n",
       "2                 3198  \n",
       "3                  230  \n",
       "4                  871  \n",
       "...                ...  \n",
       "65068               88  \n",
       "65069               47  \n",
       "65070              300  \n",
       "65071               42  \n",
       "65072               77  \n",
       "\n",
       "[65073 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fieldinfo = myoa.fieldinfo\n",
    "fieldinfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0d512670",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "lvl2 = np.sort(fieldinfo[fieldinfo['FieldLevel'] == 2]['FieldId'].values)\n",
    "\n",
    "pubtopics2 = pub2field[pyscisci.isin_sorted(pub2field['FieldId'].values, lvl2)].merge(pub2country, how='left', on='PublicationId')\n",
    "pubtopics2['Year'] = pubtopics2['PublicationId'].map(pub2year)\n",
    "\n",
    "global_topic_div = pubtopics2.groupby(['Year'], as_index=False)['FieldId'].apply(pyscisci.simpson)\n",
    "global_topic_div.rename(columns={'FieldId':'TopicDiversity'}, inplace=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4d8b6b8a",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "global_topic_div.to_csv(os.path.join(path2countrydata, \"oa_global_topic2_diversity.csv.gz\"),\n",
    "                        compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "41d08667",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pubtopics2_div = pubtopics2.groupby(['Country', 'Year'], as_index=False)['FieldId'].apply(pyscisci.simpson)\n",
    "pubtopics2_div.rename(columns={'FieldId':'TopicDiversity'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "72a873f7",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "pubtopics2_div.to_csv(os.path.join(path2countrydata, \"oa_country_topic2_diversity.csv.gz\"),\n",
    "                        compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7bad51f4",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(322054451, 4)\n",
      "(16669, 3)\n"
     ]
    }
   ],
   "source": [
    "lvl1= np.sort(fieldinfo[fieldinfo['FieldLevel'] == 1]['FieldId'].values)\n",
    "\n",
    "pubtopics1 = pub2field[pyscisci.isin_sorted(pub2field['FieldId'].values, lvl1)].merge(pub2country, how='left', on='PublicationId')\n",
    "pubtopics1['Year'] = pubtopics1['PublicationId'].map(pub2year)\n",
    "print(pubtopics1.shape)\n",
    "\n",
    "pubtopics1_div = pubtopics1.groupby(['Country', 'Year'], as_index=False)['FieldId'].apply(pyscisci.simpson)\n",
    "pubtopics1_div.rename(columns={'FieldId':'TopicDiversity'}, inplace=True)\n",
    "\n",
    "pubtopics1_div.to_csv(os.path.join(path2countrydata, \"oa_country_topic1_diversity.csv.gz\"),\n",
    "                        compression='gzip', mode='w', header=True, index=False)\n",
    "print(pubtopics1_div.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6744a72e",
   "metadata": {},
   "source": [
    "# Top Journals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1aeab9b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "pub2country = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"))\n",
    "c5 = pd.read_csv(os.path.join(path2countrydata, \"oa_c5.csv.gz\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "731a4e6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pub = myoa.load_publications(columns =['PublicationId', 'Year', 'DocType', 'JournalId'])\n",
    "pub = pub.merge(pub2country, how='left', on='PublicationId')\n",
    "pub = pub.merge(c5[['CitedPublicationId', 'C5']].drop_duplicates(), how='left', \n",
    "                left_on='PublicationId', right_on='CitedPublicationId')\n",
    "\n",
    "pub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74b29b3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "kept_doctypes = np.sort(['journal-article', 'proceedings-article'])\n",
    "pub.dropna(inplace=True)\n",
    "print(pub.shape)\n",
    "pub = pub[pyscisci.isin_sorted(pub['DocType'].values, kept_doctypes)].reset_index(drop=True)\n",
    "print(pub.shape)\n",
    "pub = pub[pub['Year'] >= 1900].reset_index(drop=True)\n",
    "print(pub.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec5d4383",
   "metadata": {},
   "outputs": [],
   "source": [
    "pub['logc5'] = np.log(pub['C5'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45d135b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "fieldinfo = myoa.fieldinfo\n",
    "lvl1 = np.sort(fieldinfo[fieldinfo['FieldLevel'] == 1]['FieldId'].unique())\n",
    "pub2field = myoa.load_pub2field(columns=['PublicationId', 'FieldId'], filter_dict={'FieldId':lvl1})\n",
    "\n",
    "pub = pub.merge(pub2field, how='left', on='PublicationId').dropna()\n",
    "pub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "28cd4421",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>FieldId</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>logc5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>2524010</td>\n",
       "      <td>1.056378e+08</td>\n",
       "      <td>0.231049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>2524010</td>\n",
       "      <td>4.210229e+09</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>10138342</td>\n",
       "      <td>4.210229e+09</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>11171543</td>\n",
       "      <td>4.210203e+09</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>13965031</td>\n",
       "      <td>1.056378e+08</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170126</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.693147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170127</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>2.079442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170128</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>1.151293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170129</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170130</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306535e+09</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19170131 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Year     FieldId     JournalId     logc5\n",
       "0         1900.0     2524010  1.056378e+08  0.231049\n",
       "1         1900.0     2524010  4.210229e+09  0.000000\n",
       "2         1900.0    10138342  4.210229e+09  0.000000\n",
       "3         1900.0    11171543  4.210203e+09  0.000000\n",
       "4         1900.0    13965031  1.056378e+08  0.000000\n",
       "...          ...         ...           ...       ...\n",
       "19170126  2017.0  2522767166  4.306531e+09  0.693147\n",
       "19170127  2017.0  2522767166  4.306531e+09  2.079442\n",
       "19170128  2017.0  2522767166  4.306531e+09  1.151293\n",
       "19170129  2017.0  2522767166  4.306531e+09  0.000000\n",
       "19170130  2017.0  2522767166  4.306535e+09  0.000000\n",
       "\n",
       "[19170131 rows x 4 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanlogcite = pub.groupby(['Year', 'FieldId', 'JournalId'], as_index=False)['logc5'].mean()\n",
    "meanlogcite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "01bc2a8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "meanlogcite.to_csv(os.path.join(path2countrydata, 'oa_journal_field_meanlogc5.csv.gz'), index=False, \n",
    "                   mode='w', header=True, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0d332495",
   "metadata": {},
   "outputs": [],
   "source": [
    "meanlogcite17 = meanlogcite[meanlogcite['Year'] == 2017].rese\n",
    "njournals = meanlogcite17.groupby('FieldId')['JournalId'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "66dd9267",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/69/w28tw6xx26nd7x4yv_sh2mgm0000gp/T/ipykernel_24179/2185435163.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  meanlogcite17['JournalRank'] = meanlogcite17.groupby('FieldId')['logc5'].transform(lambda a: pyscisci.rank_array(a, ascending=False, normed=False))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>FieldId</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>logc5</th>\n",
       "      <th>JournalRank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18025090</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>2.677290e+05</td>\n",
       "      <td>2.663329</td>\n",
       "      <td>270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18025091</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>3.232370e+05</td>\n",
       "      <td>2.772589</td>\n",
       "      <td>226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18025092</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>6.306940e+05</td>\n",
       "      <td>1.329661</td>\n",
       "      <td>919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18025093</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>6.789650e+05</td>\n",
       "      <td>0.577623</td>\n",
       "      <td>1386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18025094</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>8.153010e+05</td>\n",
       "      <td>2.993383</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170126</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>3497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170127</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>2.079442</td>\n",
       "      <td>1830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170128</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>1.151293</td>\n",
       "      <td>2934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170129</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170130</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306535e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4498</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1145041 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Year     FieldId     JournalId     logc5  JournalRank\n",
       "18025090  2017.0       40700  2.677290e+05  2.663329          270\n",
       "18025091  2017.0       40700  3.232370e+05  2.772589          226\n",
       "18025092  2017.0       40700  6.306940e+05  1.329661          919\n",
       "18025093  2017.0       40700  6.789650e+05  0.577623         1386\n",
       "18025094  2017.0       40700  8.153010e+05  2.993383          171\n",
       "...          ...         ...           ...       ...          ...\n",
       "19170126  2017.0  2522767166  4.306531e+09  0.693147         3497\n",
       "19170127  2017.0  2522767166  4.306531e+09  2.079442         1830\n",
       "19170128  2017.0  2522767166  4.306531e+09  1.151293         2934\n",
       "19170129  2017.0  2522767166  4.306531e+09  0.000000         4251\n",
       "19170130  2017.0  2522767166  4.306535e+09  0.000000         4498\n",
       "\n",
       "[1145041 rows x 5 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanlogcite17['JournalRank'] = meanlogcite17.groupby('FieldId')['logc5'].transform(lambda a: pyscisci.rank_array(a, ascending=False, normed=False))\n",
    "\n",
    "meanlogcite17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ce3d77d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14200\n"
     ]
    }
   ],
   "source": [
    "print(meanlogcite17['FieldId'].nunique()*50)\n",
    "top_journals = np.sort(meanlogcite17[meanlogcite17['JournalRank'] < 50]['JournalId'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0a4cc69c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>FieldId</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>logc5</th>\n",
       "      <th>JournalRank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>2524010</td>\n",
       "      <td>1.056378e+08</td>\n",
       "      <td>0.231049</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>2524010</td>\n",
       "      <td>4.210229e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>10138342</td>\n",
       "      <td>4.210229e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>11171543</td>\n",
       "      <td>4.210203e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1900.0</td>\n",
       "      <td>13965031</td>\n",
       "      <td>1.056378e+08</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170126</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>3497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170127</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>2.079442</td>\n",
       "      <td>1830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170128</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>1.151293</td>\n",
       "      <td>2934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170129</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170130</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306535e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4498</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19170131 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Year     FieldId     JournalId     logc5  JournalRank\n",
       "0         1900.0     2524010  1.056378e+08  0.231049            0\n",
       "1         1900.0     2524010  4.210229e+09  0.000000            1\n",
       "2         1900.0    10138342  4.210229e+09  0.000000            0\n",
       "3         1900.0    11171543  4.210203e+09  0.000000            0\n",
       "4         1900.0    13965031  1.056378e+08  0.000000            1\n",
       "...          ...         ...           ...       ...          ...\n",
       "19170126  2017.0  2522767166  4.306531e+09  0.693147         3497\n",
       "19170127  2017.0  2522767166  4.306531e+09  2.079442         1830\n",
       "19170128  2017.0  2522767166  4.306531e+09  1.151293         2934\n",
       "19170129  2017.0  2522767166  4.306531e+09  0.000000         4251\n",
       "19170130  2017.0  2522767166  4.306535e+09  0.000000         4498\n",
       "\n",
       "[19170131 rows x 5 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanlogcite['JournalRank'] = meanlogcite.groupby(['Year', 'FieldId'])['logc5'].transform(lambda a: pyscisci.rank_array(a, ascending=False, normed=False))\n",
    "\n",
    "meanlogcite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a89aadf3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>FieldId</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>logc5</th>\n",
       "      <th>JournalRank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2581707</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>1.234856e+06</td>\n",
       "      <td>1.074269</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581708</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>1.558358e+06</td>\n",
       "      <td>0.888939</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581709</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>2.228914e+06</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581710</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>2.876017e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581711</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>3.416249e+06</td>\n",
       "      <td>0.541610</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170126</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>3497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170127</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>2.079442</td>\n",
       "      <td>1830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170128</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>1.151293</td>\n",
       "      <td>2934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170129</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306531e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19170130</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306535e+09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4498</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16588424 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Year     FieldId     JournalId     logc5  JournalRank\n",
       "2581707   1990.0       40700  1.234856e+06  1.074269           84\n",
       "2581708   1990.0       40700  1.558358e+06  0.888939           98\n",
       "2581709   1990.0       40700  2.228914e+06  0.693147          119\n",
       "2581710   1990.0       40700  2.876017e+06  0.000000          171\n",
       "2581711   1990.0       40700  3.416249e+06  0.541610          154\n",
       "...          ...         ...           ...       ...          ...\n",
       "19170126  2017.0  2522767166  4.306531e+09  0.693147         3497\n",
       "19170127  2017.0  2522767166  4.306531e+09  2.079442         1830\n",
       "19170128  2017.0  2522767166  4.306531e+09  1.151293         2934\n",
       "19170129  2017.0  2522767166  4.306531e+09  0.000000         4251\n",
       "19170130  2017.0  2522767166  4.306535e+09  0.000000         4498\n",
       "\n",
       "[16588424 rows x 5 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanlogcite = meanlogcite[meanlogcite['Year'] >= 1990]\n",
    "meanlogcite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5d764c99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>FieldId</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>logc5</th>\n",
       "      <th>JournalRank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2581713</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>3.880285e+06</td>\n",
       "      <td>2.498514</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581716</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>5.353659e+06</td>\n",
       "      <td>1.945910</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581718</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>5.984737e+06</td>\n",
       "      <td>1.945910</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581721</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>9.731383e+06</td>\n",
       "      <td>1.519198</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2581731</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>40700</td>\n",
       "      <td>2.607397e+07</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19168790</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.210178e+09</td>\n",
       "      <td>5.293305</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19168830</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.210182e+09</td>\n",
       "      <td>5.641907</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19168910</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.210190e+09</td>\n",
       "      <td>6.711740</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19169237</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.210230e+09</td>\n",
       "      <td>6.259581</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19169410</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2522767166</td>\n",
       "      <td>4.306418e+09</td>\n",
       "      <td>5.710427</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>396942 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Year     FieldId     JournalId     logc5  JournalRank\n",
       "2581713   1990.0       40700  3.880285e+06  2.498514            7\n",
       "2581716   1990.0       40700  5.353659e+06  1.945910           27\n",
       "2581718   1990.0       40700  5.984737e+06  1.945910           25\n",
       "2581721   1990.0       40700  9.731383e+06  1.519198           45\n",
       "2581731   1990.0       40700  2.607397e+07  1.609438           42\n",
       "...          ...         ...           ...       ...          ...\n",
       "19168790  2017.0  2522767166  4.210178e+09  5.293305           39\n",
       "19168830  2017.0  2522767166  4.210182e+09  5.641907           19\n",
       "19168910  2017.0  2522767166  4.210190e+09  6.711740            1\n",
       "19169237  2017.0  2522767166  4.210230e+09  6.259581            4\n",
       "19169410  2017.0  2522767166  4.306418e+09  5.710427           16\n",
       "\n",
       "[396942 rows x 5 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yearly_top_journals = meanlogcite[meanlogcite['JournalRank'] < 50]\n",
    "yearly_top_journals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "284de57c",
   "metadata": {},
   "outputs": [],
   "source": [
    "yearly_top_journals.to_csv('oa_yearly_top_journals.csv.gz', index=False, \n",
    "                   mode='w', header=True, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4f9a6d16",
   "metadata": {},
   "outputs": [],
   "source": [
    "yearly_top_journals = yearly_top_journals[['Year', 'JournalId']].drop_duplicates()\n",
    "yearly_top_journals['TopJournal'] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fda1f16a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13c680eb98b240f2b5cb37a5297c113c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publications:   0%|          | 0/513 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (9,14) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (9) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (9,10) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n",
      "/Users/hgt6rn/Documents/GitHub/pyscisci/pyscisci/datasource/readwrite.py:155: DtypeWarning: Columns (8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  return pd.read_csv(fname, compression='gzip')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(63898746, 3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "country_pub = np.sort(pub2country['PublicationId'].unique())\n",
    "pub2 = myoa.load_publications(columns =['PublicationId', 'Year', 'JournalId'], filter_dict = {'PublicationId':country_pub})\n",
    "pub2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0f17868a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Year</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>TopJournal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3147477878</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>4.210191e+09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4205868985</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1.062967e+08</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4225966262</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>1.404626e+07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2471332289</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>2.754768e+09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2367225558</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>2.070659e+07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57483354</th>\n",
       "      <td>3022241891</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>1.419533e+08</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57483355</th>\n",
       "      <td>2396240571</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>2.755648e+09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57483356</th>\n",
       "      <td>2358301002</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>2.764507e+09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57483357</th>\n",
       "      <td>2953065950</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>4.210235e+09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57483358</th>\n",
       "      <td>2166352754</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>2.751447e+09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>57483359 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          PublicationId    Year     JournalId  TopJournal\n",
       "0            3147477878  2021.0  4.210191e+09         0.0\n",
       "1            4205868985  2019.0  1.062967e+08         0.0\n",
       "2            4225966262  2022.0  1.404626e+07         0.0\n",
       "3            2471332289  2006.0  2.754768e+09         0.0\n",
       "4            2367225558  2005.0  2.070659e+07         0.0\n",
       "...                 ...     ...           ...         ...\n",
       "57483354     3022241891  2020.0  1.419533e+08         0.0\n",
       "57483355     2396240571  2012.0  2.755648e+09         0.0\n",
       "57483356     2358301002  2009.0  2.764507e+09         0.0\n",
       "57483357     2953065950  2020.0  4.210235e+09         0.0\n",
       "57483358     2166352754  2006.0  2.751447e+09         0.0\n",
       "\n",
       "[57483359 rows x 4 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub2 = pub2[pub2['Year']>=1990]\n",
    "pub2 = pub2.merge(yearly_top_journals[['Year', 'JournalId', 'TopJournal']], how='left', on=['Year', 'JournalId'])\n",
    "pub2.fillna(0, inplace=True)\n",
    "pub2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3f3f39b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>Year</th>\n",
       "      <th>JournalId</th>\n",
       "      <th>TopJournal</th>\n",
       "      <th>Country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3147477878</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>4.210191e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>BR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4205868985</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1.062967e+08</td>\n",
       "      <td>0.0</td>\n",
       "      <td>FR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4225966262</td>\n",
       "      <td>2022.0</td>\n",
       "      <td>1.404626e+07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>BE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2471332289</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>2.754768e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2367225558</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>2.070659e+07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69754568</th>\n",
       "      <td>3022241891</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>1.419533e+08</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69754569</th>\n",
       "      <td>2396240571</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>2.755648e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69754570</th>\n",
       "      <td>2358301002</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>2.764507e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>CN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69754571</th>\n",
       "      <td>2953065950</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>4.210235e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69754572</th>\n",
       "      <td>2166352754</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>2.751447e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TH</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69754573 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          PublicationId    Year     JournalId  TopJournal Country\n",
       "0            3147477878  2021.0  4.210191e+09         0.0      BR\n",
       "1            4205868985  2019.0  1.062967e+08         0.0      FR\n",
       "2            4225966262  2022.0  1.404626e+07         0.0      BE\n",
       "3            2471332289  2006.0  2.754768e+09         0.0      CN\n",
       "4            2367225558  2005.0  2.070659e+07         0.0      CN\n",
       "...                 ...     ...           ...         ...     ...\n",
       "69754568     3022241891  2020.0  1.419533e+08         0.0      GB\n",
       "69754569     2396240571  2012.0  2.755648e+09         0.0      CN\n",
       "69754570     2358301002  2009.0  2.764507e+09         0.0      CN\n",
       "69754571     2953065950  2020.0  4.210235e+09         0.0      US\n",
       "69754572     2166352754  2006.0  2.751447e+09         0.0      TH\n",
       "\n",
       "[69754573 rows x 5 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub2 = pub2.merge(pub2country[['PublicationId', 'Country']], how='left', on='PublicationId')\n",
    "pub2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a39f742b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(69754573, 5)\n",
      "(69754566, 5)\n"
     ]
    }
   ],
   "source": [
    "print(pub2.shape)\n",
    "pub2.drop_duplicates(subset=['PublicationId', 'Year', 'Country'], inplace=True)\n",
    "print(pub2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "722fcde9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Country</th>\n",
       "      <th>NumPub</th>\n",
       "      <th>TopJournal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AD</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AE</td>\n",
       "      <td>109</td>\n",
       "      <td>73.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AF</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AL</td>\n",
       "      <td>34</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>AM</td>\n",
       "      <td>212</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7222</th>\n",
       "      <td>2121.0</td>\n",
       "      <td>US</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7223</th>\n",
       "      <td>2121.0</td>\n",
       "      <td>ZA</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7224</th>\n",
       "      <td>2200.0</td>\n",
       "      <td>CY</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7225</th>\n",
       "      <td>2202.0</td>\n",
       "      <td>FR</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7226</th>\n",
       "      <td>2202.0</td>\n",
       "      <td>MD</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7227 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Year Country  NumPub  TopJournal\n",
       "0     1990.0      AD       1         0.0\n",
       "1     1990.0      AE     109        73.0\n",
       "2     1990.0      AF       2         1.0\n",
       "3     1990.0      AL      34        10.0\n",
       "4     1990.0      AM     212        96.0\n",
       "...      ...     ...     ...         ...\n",
       "7222  2121.0      US       1         0.0\n",
       "7223  2121.0      ZA      10         0.0\n",
       "7224  2200.0      CY       1         0.0\n",
       "7225  2202.0      FR       1         0.0\n",
       "7226  2202.0      MD       1         0.0\n",
       "\n",
       "[7227 rows x 4 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totalprod = pub2.groupby(['Year', 'Country'], as_index=False)['PublicationId'].nunique().rename(columns={'PublicationId':'NumPub'})\n",
    "numtop = pub2.groupby(['Year', 'Country'], as_index=False)['TopJournal'].sum()\n",
    "\n",
    "countryprod = totalprod.merge(numtop, on=['Year', 'Country'])\n",
    "countryprod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "3ea8af3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "countryprod['FracTop'] = countryprod['TopJournal'] / countryprod['NumPub'] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "9b5e0990",
   "metadata": {},
   "outputs": [],
   "source": [
    "countryprod.to_csv(os.path.join(path2countrydata, \"oa_countrytopproductivity.csv.gz\"), compression='gzip', mode='w', header=True, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4c926003",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>NumPub</th>\n",
       "      <th>TopJournal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>534873</td>\n",
       "      <td>339137.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1991.0</td>\n",
       "      <td>574290</td>\n",
       "      <td>354427.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1992.0</td>\n",
       "      <td>598442</td>\n",
       "      <td>366369.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1993.0</td>\n",
       "      <td>626750</td>\n",
       "      <td>385591.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1994.0</td>\n",
       "      <td>657471</td>\n",
       "      <td>391319.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1995.0</td>\n",
       "      <td>681781</td>\n",
       "      <td>407563.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1996.0</td>\n",
       "      <td>740613</td>\n",
       "      <td>431591.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>773262</td>\n",
       "      <td>440247.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1998.0</td>\n",
       "      <td>807327</td>\n",
       "      <td>453648.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1999.0</td>\n",
       "      <td>843503</td>\n",
       "      <td>463412.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2000.0</td>\n",
       "      <td>917252</td>\n",
       "      <td>491277.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2001.0</td>\n",
       "      <td>955152</td>\n",
       "      <td>499851.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2002.0</td>\n",
       "      <td>1024821</td>\n",
       "      <td>517723.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2003.0</td>\n",
       "      <td>1125275</td>\n",
       "      <td>540675.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2004.0</td>\n",
       "      <td>1276789</td>\n",
       "      <td>598087.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2005.0</td>\n",
       "      <td>1432182</td>\n",
       "      <td>633430.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2006.0</td>\n",
       "      <td>1631975</td>\n",
       "      <td>687041.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2007.0</td>\n",
       "      <td>1792624</td>\n",
       "      <td>710856.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2008.0</td>\n",
       "      <td>1944496</td>\n",
       "      <td>759296.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2009.0</td>\n",
       "      <td>2112927</td>\n",
       "      <td>764516.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2010.0</td>\n",
       "      <td>2247687</td>\n",
       "      <td>776978.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2011.0</td>\n",
       "      <td>2389730</td>\n",
       "      <td>840131.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2012.0</td>\n",
       "      <td>2469483</td>\n",
       "      <td>894411.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2013.0</td>\n",
       "      <td>2607081</td>\n",
       "      <td>939633.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2014.0</td>\n",
       "      <td>2674409</td>\n",
       "      <td>963165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2015.0</td>\n",
       "      <td>2710264</td>\n",
       "      <td>1019164.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2016.0</td>\n",
       "      <td>2709421</td>\n",
       "      <td>1047772.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2803995</td>\n",
       "      <td>1072119.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2018.0</td>\n",
       "      <td>2819272</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2019.0</td>\n",
       "      <td>3060728</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2020.0</td>\n",
       "      <td>3352091</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2021.0</td>\n",
       "      <td>3743723</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2022.0</td>\n",
       "      <td>2826602</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2023.0</td>\n",
       "      <td>16586</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2024.0</td>\n",
       "      <td>292</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2025.0</td>\n",
       "      <td>15</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2026.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2027.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2028.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2029.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2030.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>2032.0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>2038.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>2051.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>2061.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>2071.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>2077.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>2088.0</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>2091.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>2100.0</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>2103.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>2104.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>2105.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>2106.0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>2107.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>2109.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>2115.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>2121.0</td>\n",
       "      <td>30</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>2200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>2202.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Year   NumPub  TopJournal\n",
       "0   1990.0   534873    339137.0\n",
       "1   1991.0   574290    354427.0\n",
       "2   1992.0   598442    366369.0\n",
       "3   1993.0   626750    385591.0\n",
       "4   1994.0   657471    391319.0\n",
       "5   1995.0   681781    407563.0\n",
       "6   1996.0   740613    431591.0\n",
       "7   1997.0   773262    440247.0\n",
       "8   1998.0   807327    453648.0\n",
       "9   1999.0   843503    463412.0\n",
       "10  2000.0   917252    491277.0\n",
       "11  2001.0   955152    499851.0\n",
       "12  2002.0  1024821    517723.0\n",
       "13  2003.0  1125275    540675.0\n",
       "14  2004.0  1276789    598087.0\n",
       "15  2005.0  1432182    633430.0\n",
       "16  2006.0  1631975    687041.0\n",
       "17  2007.0  1792624    710856.0\n",
       "18  2008.0  1944496    759296.0\n",
       "19  2009.0  2112927    764516.0\n",
       "20  2010.0  2247687    776978.0\n",
       "21  2011.0  2389730    840131.0\n",
       "22  2012.0  2469483    894411.0\n",
       "23  2013.0  2607081    939633.0\n",
       "24  2014.0  2674409    963165.0\n",
       "25  2015.0  2710264   1019164.0\n",
       "26  2016.0  2709421   1047772.0\n",
       "27  2017.0  2803995   1072119.0\n",
       "28  2018.0  2819272         0.0\n",
       "29  2019.0  3060728         0.0\n",
       "30  2020.0  3352091         0.0\n",
       "31  2021.0  3743723         0.0\n",
       "32  2022.0  2826602         0.0\n",
       "33  2023.0    16586         0.0\n",
       "34  2024.0      292         0.0\n",
       "35  2025.0       15         0.0\n",
       "36  2026.0        2         0.0\n",
       "37  2027.0        3         0.0\n",
       "38  2028.0        3         0.0\n",
       "39  2029.0        4         0.0\n",
       "40  2030.0        4         0.0\n",
       "41  2032.0       62         0.0\n",
       "42  2038.0        2         0.0\n",
       "43  2051.0        1         0.0\n",
       "44  2061.0        1         0.0\n",
       "45  2071.0        1         0.0\n",
       "46  2077.0        1         0.0\n",
       "47  2088.0        9         0.0\n",
       "48  2091.0        1         0.0\n",
       "49  2100.0       20         0.0\n",
       "50  2103.0        4         0.0\n",
       "51  2104.0        6         0.0\n",
       "52  2105.0        1         0.0\n",
       "53  2106.0        5         0.0\n",
       "54  2107.0        1         0.0\n",
       "55  2109.0        6         0.0\n",
       "56  2115.0        1         0.0\n",
       "57  2121.0       30         0.0\n",
       "58  2200.0        1         0.0\n",
       "59  2202.0        1         0.0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oasize = pub2.drop_duplicates(subset=['PublicationId']).groupby(['Year'], as_index=False)['PublicationId'].nunique().rename(columns={'PublicationId':'NumPub'})\n",
    "numtop = pub2.drop_duplicates(subset=['PublicationId']).groupby(['Year'], as_index=False)['TopJournal'].sum()\n",
    "\n",
    "oasize = oasize.merge(numtop, how='left', on='Year')\n",
    "oasize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "0ac6d093",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>NumPub</th>\n",
       "      <th>TopJournal</th>\n",
       "      <th>FracTop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>534873</td>\n",
       "      <td>339137.0</td>\n",
       "      <td>0.634051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1991.0</td>\n",
       "      <td>574290</td>\n",
       "      <td>354427.0</td>\n",
       "      <td>0.617157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1992.0</td>\n",
       "      <td>598442</td>\n",
       "      <td>366369.0</td>\n",
       "      <td>0.612205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1993.0</td>\n",
       "      <td>626750</td>\n",
       "      <td>385591.0</td>\n",
       "      <td>0.615223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1994.0</td>\n",
       "      <td>657471</td>\n",
       "      <td>391319.0</td>\n",
       "      <td>0.595188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1995.0</td>\n",
       "      <td>681781</td>\n",
       "      <td>407563.0</td>\n",
       "      <td>0.597792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1996.0</td>\n",
       "      <td>740613</td>\n",
       "      <td>431591.0</td>\n",
       "      <td>0.582748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>773262</td>\n",
       "      <td>440247.0</td>\n",
       "      <td>0.569337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1998.0</td>\n",
       "      <td>807327</td>\n",
       "      <td>453648.0</td>\n",
       "      <td>0.561914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1999.0</td>\n",
       "      <td>843503</td>\n",
       "      <td>463412.0</td>\n",
       "      <td>0.549390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2000.0</td>\n",
       "      <td>917252</td>\n",
       "      <td>491277.0</td>\n",
       "      <td>0.535597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2001.0</td>\n",
       "      <td>955152</td>\n",
       "      <td>499851.0</td>\n",
       "      <td>0.523321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2002.0</td>\n",
       "      <td>1024821</td>\n",
       "      <td>517723.0</td>\n",
       "      <td>0.505184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2003.0</td>\n",
       "      <td>1125275</td>\n",
       "      <td>540675.0</td>\n",
       "      <td>0.480483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2004.0</td>\n",
       "      <td>1276789</td>\n",
       "      <td>598087.0</td>\n",
       "      <td>0.468431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2005.0</td>\n",
       "      <td>1432182</td>\n",
       "      <td>633430.0</td>\n",
       "      <td>0.442283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2006.0</td>\n",
       "      <td>1631975</td>\n",
       "      <td>687041.0</td>\n",
       "      <td>0.420987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2007.0</td>\n",
       "      <td>1792624</td>\n",
       "      <td>710856.0</td>\n",
       "      <td>0.396545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2008.0</td>\n",
       "      <td>1944496</td>\n",
       "      <td>759296.0</td>\n",
       "      <td>0.390485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2009.0</td>\n",
       "      <td>2112927</td>\n",
       "      <td>764516.0</td>\n",
       "      <td>0.361828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2010.0</td>\n",
       "      <td>2247687</td>\n",
       "      <td>776978.0</td>\n",
       "      <td>0.345679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2011.0</td>\n",
       "      <td>2389730</td>\n",
       "      <td>840131.0</td>\n",
       "      <td>0.351559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2012.0</td>\n",
       "      <td>2469483</td>\n",
       "      <td>894411.0</td>\n",
       "      <td>0.362186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2013.0</td>\n",
       "      <td>2607081</td>\n",
       "      <td>939633.0</td>\n",
       "      <td>0.360416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2014.0</td>\n",
       "      <td>2674409</td>\n",
       "      <td>963165.0</td>\n",
       "      <td>0.360141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2015.0</td>\n",
       "      <td>2710264</td>\n",
       "      <td>1019164.0</td>\n",
       "      <td>0.376039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2016.0</td>\n",
       "      <td>2709421</td>\n",
       "      <td>1047772.0</td>\n",
       "      <td>0.386714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2803995</td>\n",
       "      <td>1072119.0</td>\n",
       "      <td>0.382354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2018.0</td>\n",
       "      <td>2819272</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2019.0</td>\n",
       "      <td>3060728</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>2020.0</td>\n",
       "      <td>3352091</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2021.0</td>\n",
       "      <td>3743723</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>2022.0</td>\n",
       "      <td>2826602</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>2023.0</td>\n",
       "      <td>16586</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>2024.0</td>\n",
       "      <td>292</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2025.0</td>\n",
       "      <td>15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2026.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>2027.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>2028.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2029.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2030.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>2032.0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>2038.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>2051.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>2061.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>2071.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>2077.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>2088.0</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>2091.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>2100.0</td>\n",
       "      <td>20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>2103.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>2104.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>2105.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>2106.0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>2107.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>2109.0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>2115.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>2121.0</td>\n",
       "      <td>30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>2200.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>2202.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Year   NumPub  TopJournal   FracTop\n",
       "0   1990.0   534873    339137.0  0.634051\n",
       "1   1991.0   574290    354427.0  0.617157\n",
       "2   1992.0   598442    366369.0  0.612205\n",
       "3   1993.0   626750    385591.0  0.615223\n",
       "4   1994.0   657471    391319.0  0.595188\n",
       "5   1995.0   681781    407563.0  0.597792\n",
       "6   1996.0   740613    431591.0  0.582748\n",
       "7   1997.0   773262    440247.0  0.569337\n",
       "8   1998.0   807327    453648.0  0.561914\n",
       "9   1999.0   843503    463412.0  0.549390\n",
       "10  2000.0   917252    491277.0  0.535597\n",
       "11  2001.0   955152    499851.0  0.523321\n",
       "12  2002.0  1024821    517723.0  0.505184\n",
       "13  2003.0  1125275    540675.0  0.480483\n",
       "14  2004.0  1276789    598087.0  0.468431\n",
       "15  2005.0  1432182    633430.0  0.442283\n",
       "16  2006.0  1631975    687041.0  0.420987\n",
       "17  2007.0  1792624    710856.0  0.396545\n",
       "18  2008.0  1944496    759296.0  0.390485\n",
       "19  2009.0  2112927    764516.0  0.361828\n",
       "20  2010.0  2247687    776978.0  0.345679\n",
       "21  2011.0  2389730    840131.0  0.351559\n",
       "22  2012.0  2469483    894411.0  0.362186\n",
       "23  2013.0  2607081    939633.0  0.360416\n",
       "24  2014.0  2674409    963165.0  0.360141\n",
       "25  2015.0  2710264   1019164.0  0.376039\n",
       "26  2016.0  2709421   1047772.0  0.386714\n",
       "27  2017.0  2803995   1072119.0  0.382354\n",
       "28  2018.0  2819272         0.0  0.000000\n",
       "29  2019.0  3060728         0.0  0.000000\n",
       "30  2020.0  3352091         0.0  0.000000\n",
       "31  2021.0  3743723         0.0  0.000000\n",
       "32  2022.0  2826602         0.0  0.000000\n",
       "33  2023.0    16586         0.0  0.000000\n",
       "34  2024.0      292         0.0  0.000000\n",
       "35  2025.0       15         0.0  0.000000\n",
       "36  2026.0        2         0.0  0.000000\n",
       "37  2027.0        3         0.0  0.000000\n",
       "38  2028.0        3         0.0  0.000000\n",
       "39  2029.0        4         0.0  0.000000\n",
       "40  2030.0        4         0.0  0.000000\n",
       "41  2032.0       62         0.0  0.000000\n",
       "42  2038.0        2         0.0  0.000000\n",
       "43  2051.0        1         0.0  0.000000\n",
       "44  2061.0        1         0.0  0.000000\n",
       "45  2071.0        1         0.0  0.000000\n",
       "46  2077.0        1         0.0  0.000000\n",
       "47  2088.0        9         0.0  0.000000\n",
       "48  2091.0        1         0.0  0.000000\n",
       "49  2100.0       20         0.0  0.000000\n",
       "50  2103.0        4         0.0  0.000000\n",
       "51  2104.0        6         0.0  0.000000\n",
       "52  2105.0        1         0.0  0.000000\n",
       "53  2106.0        5         0.0  0.000000\n",
       "54  2107.0        1         0.0  0.000000\n",
       "55  2109.0        6         0.0  0.000000\n",
       "56  2115.0        1         0.0  0.000000\n",
       "57  2121.0       30         0.0  0.000000\n",
       "58  2200.0        1         0.0  0.000000\n",
       "59  2202.0        1         0.0  0.000000"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oasize['FracTop'] = oasize['TopJournal'] / oasize['NumPub'] \n",
    "oasize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "47cc1514",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>OANumPub</th>\n",
       "      <th>OATopJournal</th>\n",
       "      <th>OAFracTop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1990.0</td>\n",
       "      <td>534873</td>\n",
       "      <td>339137.0</td>\n",
       "      <td>0.634051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1991.0</td>\n",
       "      <td>574290</td>\n",
       "      <td>354427.0</td>\n",
       "      <td>0.617157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1992.0</td>\n",
       "      <td>598442</td>\n",
       "      <td>366369.0</td>\n",
       "      <td>0.612205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1993.0</td>\n",
       "      <td>626750</td>\n",
       "      <td>385591.0</td>\n",
       "      <td>0.615223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1994.0</td>\n",
       "      <td>657471</td>\n",
       "      <td>391319.0</td>\n",
       "      <td>0.595188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1995.0</td>\n",
       "      <td>681781</td>\n",
       "      <td>407563.0</td>\n",
       "      <td>0.597792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1996.0</td>\n",
       "      <td>740613</td>\n",
       "      <td>431591.0</td>\n",
       "      <td>0.582748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1997.0</td>\n",
       "      <td>773262</td>\n",
       "      <td>440247.0</td>\n",
       "      <td>0.569337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1998.0</td>\n",
       "      <td>807327</td>\n",
       "      <td>453648.0</td>\n",
       "      <td>0.561914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1999.0</td>\n",
       "      <td>843503</td>\n",
       "      <td>463412.0</td>\n",
       "      <td>0.549390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2000.0</td>\n",
       "      <td>917252</td>\n",
       "      <td>491277.0</td>\n",
       "      <td>0.535597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2001.0</td>\n",
       "      <td>955152</td>\n",
       "      <td>499851.0</td>\n",
       "      <td>0.523321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2002.0</td>\n",
       "      <td>1024821</td>\n",
       "      <td>517723.0</td>\n",
       "      <td>0.505184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2003.0</td>\n",
       "      <td>1125275</td>\n",
       "      <td>540675.0</td>\n",
       "      <td>0.480483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2004.0</td>\n",
       "      <td>1276789</td>\n",
       "      <td>598087.0</td>\n",
       "      <td>0.468431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2005.0</td>\n",
       "      <td>1432182</td>\n",
       "      <td>633430.0</td>\n",
       "      <td>0.442283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2006.0</td>\n",
       "      <td>1631975</td>\n",
       "      <td>687041.0</td>\n",
       "      <td>0.420987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2007.0</td>\n",
       "      <td>1792624</td>\n",
       "      <td>710856.0</td>\n",
       "      <td>0.396545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2008.0</td>\n",
       "      <td>1944496</td>\n",
       "      <td>759296.0</td>\n",
       "      <td>0.390485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2009.0</td>\n",
       "      <td>2112927</td>\n",
       "      <td>764516.0</td>\n",
       "      <td>0.361828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2010.0</td>\n",
       "      <td>2247687</td>\n",
       "      <td>776978.0</td>\n",
       "      <td>0.345679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2011.0</td>\n",
       "      <td>2389730</td>\n",
       "      <td>840131.0</td>\n",
       "      <td>0.351559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2012.0</td>\n",
       "      <td>2469483</td>\n",
       "      <td>894411.0</td>\n",
       "      <td>0.362186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2013.0</td>\n",
       "      <td>2607081</td>\n",
       "      <td>939633.0</td>\n",
       "      <td>0.360416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2014.0</td>\n",
       "      <td>2674409</td>\n",
       "      <td>963165.0</td>\n",
       "      <td>0.360141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2015.0</td>\n",
       "      <td>2710264</td>\n",
       "      <td>1019164.0</td>\n",
       "      <td>0.376039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2016.0</td>\n",
       "      <td>2709421</td>\n",
       "      <td>1047772.0</td>\n",
       "      <td>0.386714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2017.0</td>\n",
       "      <td>2803995</td>\n",
       "      <td>1072119.0</td>\n",
       "      <td>0.382354</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Year  OANumPub  OATopJournal  OAFracTop\n",
       "0   1990.0    534873      339137.0   0.634051\n",
       "1   1991.0    574290      354427.0   0.617157\n",
       "2   1992.0    598442      366369.0   0.612205\n",
       "3   1993.0    626750      385591.0   0.615223\n",
       "4   1994.0    657471      391319.0   0.595188\n",
       "5   1995.0    681781      407563.0   0.597792\n",
       "6   1996.0    740613      431591.0   0.582748\n",
       "7   1997.0    773262      440247.0   0.569337\n",
       "8   1998.0    807327      453648.0   0.561914\n",
       "9   1999.0    843503      463412.0   0.549390\n",
       "10  2000.0    917252      491277.0   0.535597\n",
       "11  2001.0    955152      499851.0   0.523321\n",
       "12  2002.0   1024821      517723.0   0.505184\n",
       "13  2003.0   1125275      540675.0   0.480483\n",
       "14  2004.0   1276789      598087.0   0.468431\n",
       "15  2005.0   1432182      633430.0   0.442283\n",
       "16  2006.0   1631975      687041.0   0.420987\n",
       "17  2007.0   1792624      710856.0   0.396545\n",
       "18  2008.0   1944496      759296.0   0.390485\n",
       "19  2009.0   2112927      764516.0   0.361828\n",
       "20  2010.0   2247687      776978.0   0.345679\n",
       "21  2011.0   2389730      840131.0   0.351559\n",
       "22  2012.0   2469483      894411.0   0.362186\n",
       "23  2013.0   2607081      939633.0   0.360416\n",
       "24  2014.0   2674409      963165.0   0.360141\n",
       "25  2015.0   2710264     1019164.0   0.376039\n",
       "26  2016.0   2709421     1047772.0   0.386714\n",
       "27  2017.0   2803995     1072119.0   0.382354"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oasize = oasize[oasize['Year']<=2017].rename(columns={'NumPub':'OANumPub', 'TopJournal':'OATopJournal', 'FracTop':\"OAFracTop\"})\n",
    "oasize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "73d70cd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "oasize.to_csv(os.path.join(path2countrydata, \"oa_globaltopjournal.csv.gz\"), compression='gzip', mode='w', header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "078911e4",
   "metadata": {},
   "source": [
    "# Process publication abstracts for ideas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4c2962d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pub = myoa.load_publications(columns=['PublicationId', 'Year', 'DocType', 'Title'],\n",
    "                      dropna = ['PublicationId', 'Year', 'DocType', 'Title'])\n",
    "pub2country = pd.read_csv(os.path.join(path2countrydata, \"oa_pub2country.csv.gz\"))\n",
    "\n",
    "pub = pub.merge(pub2country, how='inner')\n",
    "\n",
    "pub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c4efba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "idealist = pd.read_csv('data/Chengetal_idealist.csv.gz')\n",
    "ideas = idealist['id'].unique()\n",
    "print(ideas.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4873ae3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(vocabulary = ideas, ngram_range=(1,6))\n",
    "\n",
    "punct2remove = string.punctuation\n",
    "punct2remove = punct2remove.replace(\".\", \"\").replace(\",\", \"\") # don't remove period, comma\n",
    "pattern2remove = r\"[{}]\".format(punct2remove) # create the pattern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb8113f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(os.path.join(path2openalex, 'pub2idea')):\n",
    "    os.mkdir(os.path.join(path2openalex, 'pub2idea'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08f959f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pub2keyword = []\n",
    "\n",
    "path2abstracts = os.path.join(path2openalex, 'pub2abstract')\n",
    "abstract_files = [file for (dirpath, dirnames, filenames) in os.walk(path2abstracts) for file in filenames if '.gz' in file]\n",
    "\n",
    "sfile = 0\n",
    "for ifile in range(len(abstract_files)):\n",
    "    \n",
    "    pubinfo = pd.read_csv(os.path.join(path2openalex, 'publication', 'publication{}.csv.gz'.format(ifile)))\n",
    "    pubinfo = pubinfo[['PublicationId', 'Title']]\n",
    "    pubinfo = pubinfo.set_index('PublicationId')\n",
    "    \n",
    "    with gzip.open(os.path.join(path2openalex, 'pub2abstract', 'pub2abstract{}.jsonl.gz'.format(ifile))) as infile:\n",
    "        \n",
    "        for line in infile:\n",
    "            publine = json.loads(line.decode('utf8'))\n",
    "            for k, a in publine.items():\n",
    "                if not a is None and len(a) > 0:\n",
    "                    try:\n",
    "                        fullstr = str(pubinfo.loc[int(k)]['Title']) + '. ' +  pyscisci.abstractindex2text(a)\n",
    "                    except KeyError:\n",
    "                        fullstr = pyscisci.abstractindex2text(a)\n",
    "                    fullstr = fullstr.lower()\n",
    "\n",
    "                    fullstr = re.sub(r'\\d', '', fullstr)\n",
    "\n",
    "                    fullstr = fullstr.translate(str.maketrans(\"\", \"\", pattern2remove))\n",
    "                    word_tokens_stem = \" \".join([ps.stem(w) for w in word_tokenize(fullstr)])\n",
    "                    #print(fullstr)\n",
    "\n",
    "                    found_ideas = vectorizer.transform([word_tokens_stem])\n",
    "                    r, c = found_ideas.nonzero()\n",
    "\n",
    "                    for idx in c:\n",
    "                        pub2keyword.append([int(k), ideas[idx]])\n",
    "    print(ifile)\n",
    "    if ifile % 50 == 0:\n",
    "        pub2keyword2 = pd.DataFrame(pub2keyword, columns=['PublicationId', 'Idea'])\n",
    "        pub2keyword2.to_csv(os.path.join(path2openalex, 'pub2idea', 'pub2idea{}.csv.gz'.format(sfile)),\n",
    "                  mode='w', compression='gzip', header=True, index=False)\n",
    "        sfile += 1\n",
    "        pub2keyword = []\n",
    "        \n",
    "pub2keyword = pd.DataFrame(pub2keyword, columns=['PublicationId', 'Idea'])\n",
    "pub2keyword.to_csv(os.path.join(path2openalex, 'pub2idea', 'pub2idea{}.csv.gz'.format(sfile)),\n",
    "                  mode='w', compression='gzip', header=True, index=False)\n",
    "\n",
    "pub2keyword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b18bdf73",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_code_all_hidden": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
