{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ff9777b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pyvis.network import Network\n",
    "from pybliometrics.scopus import AuthorSearch\n",
    "from pybliometrics.scopus import AuthorRetrieval\n",
    "from pprint import pprint\n",
    "import gensim\n",
    "from gensim import corpora\n",
    "import itertools\n",
    "import time\n",
    "from sklearn.cluster import KMeans\n",
    "import sys\n",
    "sys.path.append(\"..\")\n",
    "from rcn_py import scopus"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "576c228d",
   "metadata": {},
   "source": [
    "## Find the author_id by the name "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b9410a3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "s = AuthorSearch('AUTHLAST(Kalverla) and AUTHFIRST(Peter)')  #todo also check Joris van Eijnatten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "de3c3ac2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search 'AUTHLAST(Kalverla) and AUTHFIRST(Peter)' yielded 1 author as of 2023-02-28:\n",
      "    Kalverla, Peter; AUTHOR_ID:57189467443 (8 document(s))\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4c397ff6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pybliometrics.scopus.author_retrieval.AuthorRetrieval at 0x7f9964992880>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AuthorRetrieval(7004691405)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "afea74fb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.scopus.com/authid/detail.uri?partnerID=HzOxMe3b&authorId=57189467443&origin=inward\n"
     ]
    }
   ],
   "source": [
    "author_id = 57189467443\n",
    "au = AuthorRetrieval(author_id)\n",
    "print(au.self_link)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "41e5159d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nld_coauthor(author_id, depth, node_retrieved):\n",
    "    au = AuthorRetrieval(author_id)\n",
    "    docs = pd.DataFrame(au.get_documents())\n",
    "    # Access to documents for the last five years\n",
    "#     new_docs = docs[(docs.coverDate > '2018')]\n",
    "    au_id = scopus.author_ids\n",
    "    \n",
    "    link = []\n",
    "    all_node = []\n",
    "\n",
    "    node_retrieved.append(author_id)\n",
    "    \n",
    "    for i in au_id:\n",
    "        coau_id = i.split(\";\")\n",
    "        coau_id = list(map(int, coau_id))\n",
    "        new_coau_id = []\n",
    "        for j in coau_id:\n",
    "            aff = AuthorRetrieval(j).affiliation_current\n",
    "            if AuthorRetrieval(j).affiliation_current:\n",
    "                # Geo-filtering \n",
    "                # todo: all the affiliations \n",
    "                if AuthorRetrieval(j).affiliation_current[0][7] == 'nld':\n",
    "                    if j not in all_node:\n",
    "                        all_node.append(j)\n",
    "                    new_coau_id.append(j)\n",
    "                    \n",
    "        sorted_new_coauid = list(map(int, new_coau_id))\n",
    "        sorted_new_coauid.sort()\n",
    "        link = link+list(itertools.combinations(sorted_new_coauid, 2))\n",
    "        # Do recursion (increase depth of the network)\n",
    "        if depth > 0:\n",
    "            for j in sorted_new_coauid:\n",
    "                if j not in node_retrieved:\n",
    "                    nld_coauthor(j, depth-1)\n",
    "    return all_node, link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3b949111",
   "metadata": {},
   "outputs": [],
   "source": [
    "node_retrieved = []\n",
    "def all_coauthor(author_id, depth, node_retrieved):\n",
    "    au = AuthorRetrieval(author_id)\n",
    "    docs = pd.DataFrame(au.get_documents())\n",
    "    # Access to documents for the last five years\n",
    "    new_docs = docs[(docs.coverDate > '2018')]\n",
    "    au_id = new_docs.author_ids\n",
    "    \n",
    "    link = []\n",
    "    all_node = []\n",
    "\n",
    "    node_retrieved.append(author_id)\n",
    "    \n",
    "    for i in au_id:\n",
    "        coau_id = i.split(\";\")\n",
    "        coau_id = list(map(int, coau_id))\n",
    "        new_coau_id = []\n",
    "        for j in coau_id:\n",
    "            aff = AuthorRetrieval(j).affiliation_current\n",
    "            if AuthorRetrieval(j).affiliation_current:\n",
    "                # Geo-filtering \n",
    "                # todo: all the affiliations \n",
    "                # if AuthorRetrieval(j).affiliation_current[0][7] == 'nld':\n",
    "                    if j not in all_node:\n",
    "                        all_node.append(j)\n",
    "                    new_coau_id.append(j)\n",
    "                    \n",
    "        sorted_new_coauid = list(map(int, new_coau_id))\n",
    "        sorted_new_coauid.sort()\n",
    "        link = link+list(itertools.combinations(sorted_new_coauid, 2))\n",
    "        # Do recursion (increase depth of the network)\n",
    "        if depth > 0:\n",
    "            for j in sorted_new_coauid:\n",
    "                if j not in node_retrieved:\n",
    "                    nld_coauthor(j, depth-1)\n",
    "    return all_node, link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c5614758",
   "metadata": {},
   "outputs": [],
   "source": [
    "node, link = all_coauthor(author_id, 0, node_retrieved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "27d45551",
   "metadata": {},
   "outputs": [],
   "source": [
    "node, link = nld_coauthor(author_id, 0, node_retrieved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "457afe62",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rolf W. Hut\n",
      "Niels Drost\n",
      "Nick C. Van De Giesen\n",
      "Ben J.C. van Werkhoven\n",
      "Banafsheh Abdollahi\n",
      "Jerom P.M. Aerts\n",
      "Thomas Albers\n",
      "Fakhereh Alidoost\n",
      "Bouwe Andela\n",
      "Jaro Camphuijsen\n",
      "Yifat Dzigan\n",
      "Ronald van Haren\n",
      "Eric W.H. Hutton\n",
      "Peter Kalverla\n",
      "Maarten A.J. van Meersbergen\n",
      "Gijs van den Oord\n",
      "Inti Pelupessy\n",
      "Stef Smeets\n",
      "Stefan Verhoeven\n",
      "Martine G. de Vos\n",
      "Berend Weel\n",
      "Mark Schelbergen\n",
      "Roland Schmehl\n",
      "Simon Watson\n",
      "Albert A.M. Holtslag\n",
      "Reinder J. Ronda\n",
      "G. J. Steeneveld\n",
      "James B. Duncan\n"
     ]
    }
   ],
   "source": [
    "for i in node:\n",
    "    print(AuthorRetrieval(i).given_name+' '+AuthorRetrieval(i).surname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8f1d6841",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_hindex(au_id):\n",
    "    au = AuthorRetrieval(au_id)\n",
    "    return au.h_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3c805fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = []\n",
    "targets = []\n",
    "weights = []\n",
    "\n",
    "for i in link:\n",
    "    sources.append(i[0])\n",
    "    targets.append(i[1])\n",
    "    weights.append(link.count(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2c5434c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pyvis network\n",
    "N = Network(height=800, width=\"100%\", bgcolor=\"#222222\", font_color=\"white\", select_menu=True)\n",
    "N.toggle_hide_edges_on_drag(False)\n",
    "N.barnes_hut()\n",
    "\n",
    "edge_data = zip(sources, targets, weights)\n",
    "\n",
    "for e in edge_data:\n",
    "    src = e[0]\n",
    "    dst = e[1]\n",
    "    w = e[2]\n",
    "\n",
    "    N.add_node(src, src, title=src, group = au_group[src])\n",
    "    N.add_node(dst, dst, title=dst, group = au_group[dst])\n",
    "    N.add_edge(src, dst, value=w)\n",
    "\n",
    "neighbor_map = N.get_adj_list()\n",
    "\n",
    "# add neighbor data to node hover data\n",
    "for node in N.nodes:\n",
    "    neighbors = []\n",
    "    for neighbor_id in neighbor_map[node[\"id\"]]:\n",
    "        neigh = AuthorRetrieval(neighbor_id)\n",
    "        neighbors.append(neigh.given_name+' '+neigh.surname)\n",
    "    #node[\"title\"] = \" Neighbors: \\n\" + \" \\n\".join(neighbors)\n",
    "    node[\"title\"] = \"Link to the author’s API page:\\n\" + AuthorRetrieval(node[\"id\"]).self_link\n",
    "    node[\"value\"] = get_hindex(node[\"id\"])\n",
    "    node[\"label\"] = AuthorRetrieval(node[\"id\"]).given_name+' '+AuthorRetrieval(node[\"id\"]).surname\n",
    "    # node[\"group\"] = group[node[\"id\"]]\n",
    "    # node[\"font_size\"] = get_hindex(node[\"id\"])\n",
    "\n",
    "N.show(\"peter.html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "142cd06e",
   "metadata": {},
   "outputs": [],
   "source": [
    "au = AuthorRetrieval(author_id)\n",
    "docs = pd.DataFrame(au.get_documents())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "050af5e6",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'57189467443'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.authors[0].eid.split('-')[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "488d2b27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_country(author_list, country_code):\n",
    "    filtered_authors = []\n",
    "    for i in author_list:\n",
    "        aff_list = AuthorRetrieval(i).affiliation_current\n",
    "        for aff_cur in aff_list:\n",
    "            if aff_cur.country_code == country_code:\n",
    "                filtered_authors.append(i)\n",
    "                break\n",
    "    return filtered_authors\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2ea2b8fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    }
   ],
   "source": [
    "author_lst = filter_country(node, 'nld')\n",
    "print(len(author_lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e0b2836",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0e9c218f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coauthor(author_first, author_last, depth):\n",
    "    s = AuthorSearch('AUTHLAST('+author_last+') and AUTHFIRST('+author_first+')')\n",
    "    author_id = s.authors[0].eid.split('-')[-1]\n",
    "    node2 = []\n",
    "    node, link = nld_coauthor(author_id, depth, node2)\n",
    "    sources = []\n",
    "    targets = []\n",
    "    weights = []\n",
    "\n",
    "    for i in link:\n",
    "        sources.append(i[0])\n",
    "        targets.append(i[1])\n",
    "        weights.append(link.count(i))\n",
    "    \n",
    "    # Pyvis network\n",
    "    N = Network(height=800, width=\"100%\", bgcolor=\"#222222\", font_color=\"white\", select_menu=True)\n",
    "    N.toggle_hide_edges_on_drag(False)\n",
    "    N.barnes_hut()\n",
    "\n",
    "    edge_data = zip(sources, targets, weights)\n",
    "\n",
    "    for e in edge_data:\n",
    "        src = e[0]\n",
    "        dst = e[1]\n",
    "        w = e[2]\n",
    "\n",
    "        N.add_node(src, src, title=src)\n",
    "        N.add_node(dst, dst, title=dst)\n",
    "        N.add_edge(src, dst, value=w)\n",
    "\n",
    "    neighbor_map = N.get_adj_list()\n",
    "\n",
    "    # add neighbor data to node hover data\n",
    "    for node in N.nodes:\n",
    "        neighbors = []\n",
    "        for neighbor_id in neighbor_map[node[\"id\"]]:\n",
    "            neigh = AuthorRetrieval(neighbor_id)\n",
    "            neighbors.append(neigh.given_name+' '+neigh.surname)\n",
    "        #node[\"title\"] = \" Neighbors: \\n\" + \" \\n\".join(neighbors)\n",
    "        node[\"title\"] = \"Link to the author’s API page:\\n\" + AuthorRetrieval(node[\"id\"]).self_link\n",
    "        node[\"value\"] = get_hindex(node[\"id\"])\n",
    "        node[\"label\"] = AuthorRetrieval(node[\"id\"]).given_name+' '+AuthorRetrieval(node[\"id\"]).surname\n",
    "       \n",
    "\n",
    "    N.show(author_last + \".html\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ad0f8a4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_coauthor(\"Bart\",\"Schilperoort\",0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f708414a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Hutton et al. (2016) argued that computational...\n",
       "1    Airborne wind energy (AWE) systems harness ene...\n",
       "2    Offshore wind energy production is rapidly gro...\n",
       "3    Ten years of ERA5 reanalysis data are combined...\n",
       "4    Numerical weather prediction models play an im...\n",
       "5    Uncertainty reduction in offshore wind systems...\n",
       "6    In numerical models of geophysical fluid syste...\n",
       "7    In the winter of 2012/13, the Katabatic Winds ...\n",
       "Name: description, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs.description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36afd39f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7022d7ae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9eb59183",
   "metadata": {},
   "source": [
    "## Scopus search by subject code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab3ac47a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pybliometrics.scopus import SerialSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5142cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = SerialSearch(query={\"subjCode\": \"2601\"})"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f4336629",
   "metadata": {},
   "source": [
    "## Get Scopus Author ID from ORCID\n",
    "\n",
    "This function solves the problem of overlap between the RSD and Scopus databases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98d4bc19",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "438f23e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "orcid = \"0000-0001-5371-5876\"\n",
    "url = \"http://api.elsevier.com/content/search/author?query=ORCID%28\"+orcid+\"%29\"\n",
    "myAPIkey = \"3d120b6ddb7d069272dfc2bc68af4028\"\n",
    "\n",
    "header = {'Accept' : 'application/json', \n",
    "          'X-ELS-APIKey' : myAPIkey}\n",
    "\n",
    "resp = requests.get(url, headers=header)\n",
    "\n",
    "results = resp.json()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "bf93070f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Casson F.J.\n"
     ]
    }
   ],
   "source": [
    "preferred_name = results['search-results']['entry'][0]['preferred-name']\n",
    "name  = preferred_name['surname'] + ' ' + preferred_name['initials']\n",
    "print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "354f6177",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26025851000\n"
     ]
    }
   ],
   "source": [
    "scopus_author_id = results['search-results']['entry'][0]['dc:identifier'].split(':')[-1]\n",
    "print(scopus_author_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ac2086e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "78d178f77aa2234760f0004d20edde8d97d9d6aae15b3a03389e75a39bac3a7d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
