{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     /Users/jennifer/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/jennifer/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/jennifer/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import sqlite3\n",
    "# import ibis\n",
    "import sys\n",
    "import pandas as pd\n",
    "sys.path.append(\"..\")\n",
    "from rcn_py import database\n",
    "from rcn_py import neo4j_scopus\n",
    "from neo4j import GraphDatabase\n",
    "import os"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SQLite DB\n",
    "    author-paper table\n",
    "    author info table\n",
    "    publication table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "con = sqlite3.connect(\"/Users/jennifer/RCSNA/data/tutorial.db\")\n",
    "cur = con.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7fc879214f80>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cur.execute(\"CREATE TABLE test_paper(doi primary key, title, abstract)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7fc879214f80>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cur.execute(\"CREATE TABLE test_authors(orcid primary key, name, country)\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7fc879214f80>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cur.execute(\"CREATE TABLE test_author_papers2(orcid, doi, primary key(orcid, doi) FOREIGN KEY(orcid) REFERENCES test_authors(orcid), FOREIGN KEY(doi) REFERENCES test_paper(doi))\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "con.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Insert data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Done.'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "database.insert_database(\"/Users/jennifer/RCSNA/data/tutorial.db\", '',\"Peter Kalverla\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "database.insert_coauthors_pub(\"/Users/jennifer/RCSNA/data/tutorial.db\", \"Peter Kalverla\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "database.insert_cocoauthors(\"/Users/jennifer/RCSNA/data/tutorial.db\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fetch data from database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "con = sqlite3.connect(\"/Users/jennifer/RCSNA/data/tutorial.db\")\n",
    "cur = con.cursor()\n",
    "\n",
    "# Topic modeling (title or abstract) to cluster papers\n",
    "clusters, idx2topics = database.pub_cluster(cur)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get coauthor relationship\n",
    "links = database.fetch_relationships(cur)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# assign author's group as the most frequent paper group\n",
    "orcid_group = database.author_cluster(cur, clusters)\n",
    "\n",
    "cur.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pyvis\n",
    "database.build_network_database(\"Database\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Store more data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "con = sqlite3.connect(\"/Users/jennifer/RCSNA/data/tutorial.db\")\n",
    "cur = con.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "con_rsd = sqlite3.connect(\"/Users/jennifer/RCSNA/data/rsd.db\")\n",
    "cur_rsd = con_rsd.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_rsd = cur_rsd.execute(\"\"\"\n",
    "SELECT orcid, given_name, family_name, affiliation \n",
    "FROM author \n",
    "WHERE affiliation = 'Netherlands eScience Center'\n",
    "GROUP BY orcid\n",
    "UNION\n",
    "SELECT orcid, given_name, family_name, affiliation \n",
    "FROM contributor \n",
    "WHERE affiliation = 'Netherlands eScience Center'\n",
    "GROUP BY orcid\n",
    "\"\"\")\n",
    "escience_member = res_rsd.fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0000-0001-5301-3276',\n",
       " 'Jesus',\n",
       " 'Garcia Gonzalez',\n",
       " 'Netherlands eScience Center')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "escience_member[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def insert_escience_rsd_member(db_path):\n",
    "    con_rsd = sqlite3.connect(\"/Users/jennifer/RCSNA/data/rsd.db\")\n",
    "    cur_rsd = con_rsd.cursor()\n",
    "    res_rsd = cur_rsd.execute(\"\"\"\n",
    "        SELECT orcid, given_name, family_name, affiliation \n",
    "        FROM author \n",
    "        WHERE affiliation = 'Netherlands eScience Center'\n",
    "        GROUP BY orcid\n",
    "        UNION\n",
    "        SELECT orcid, given_name, family_name, affiliation \n",
    "        FROM contributor \n",
    "        WHERE affiliation = 'Netherlands eScience Center'\n",
    "        GROUP BY orcid\n",
    "        \"\"\")\n",
    "    escience_members = res_rsd.fetchall()\n",
    "    for member in escience_members:\n",
    "        orcid_id = member[0]\n",
    "        fullname = member[1] + \" \" + member[2]\n",
    "        database.insert_database(db_path, orcid_id, fullname)\n",
    "\n",
    "    con_rsd.commit()\n",
    "    con_rsd.close()\n",
    "    return \"Insertion complete\"\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Insertion complete'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insert_escience_rsd_member(\"/Users/jennifer/RCSNA/data/tutorial.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cur.execute(\"DELETE FROM author_publication WHERE doi = 'None'\")\n",
    "con.commit()\n",
    "con.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remote AuraDB, only one for free'\n",
    "# uri = \"neo4j+s://a93e1a64.databases.neo4j.io\"\n",
    "# user = \"neo4j\"\n",
    "# password = \"ifVwf-BGTZt1ltrNgVbh5Q6aGSxQSksQJSxBUbJb1BE\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# local AuraDB\n",
    "uri = \"bolt://localhost:7687\"\n",
    "user = \"neo4j\"\n",
    "password = \"zhiningbai\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/fp/vghv4nhj5zg720lgbcqs7x300000gn/T/ipykernel_24418/2331392860.py:2: ExperimentalWarning: The configuration may change in the future.\n",
      "  check_verify.verify_connectivity()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Neo4j/5.6.0'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_verify =  GraphDatabase.driver(uri, auth=(user, password))\n",
    "check_verify.verify_connectivity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add constraint Cypher\n",
    "\"\"\"\n",
    "CREATE CONSTRAINT pub_doi IF NOT EXISTS\n",
    "FOR (p:Publication) REQUIRE p.doi IS UNIQUE\n",
    "\n",
    "CREATE CONSTRAINT scopus_id IF NOT EXISTS\n",
    "FOR (n:Person) REQUIRE n.scopus_id IS UNIQUE\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = GraphDatabase.driver(uri, auth=(user, password))\n",
    "session = driver.session(database=\"neo4j\")\n",
    "session.execute_write(neo4j_scopus.add_constraint) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_subject = [\"Medicine\", \"Biochemistry, Genetics and Molecular Biology\", \"Social Sciences\",\n",
    "                \"Engineering\", \"Physics and Astronomy\", \"Computer Science\", \"Environmental Science\",\n",
    "                \"Agricultural and Biological Sciences\", \"Earth and Planetary Sciences\",\n",
    "                \"Chemistry\", \"Psychology\", \"Neuroscience\", \"Mathematics\",\n",
    "                \"Immunology and Microbiology\", \"Materials Science\", \"Multidisciplinary\",\n",
    "                \"Arts and Humanities\", \"Chemical Engineering\", \"Pharmacology, Toxicology and Pharmaceutics\",\n",
    "                \"Business, Management and Accounting\", \"Energy\", \"Nursing\",\n",
    "                \"Economics, Econometrics and Finance\", \"Health Professions\",\n",
    "                \"Decision Sciences\", \"Veterinary\", \"Dentistry\"\n",
    "                ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(file_subject)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/fp/vghv4nhj5zg720lgbcqs7x300000gn/T/ipykernel_24418/3125506189.py:5: ExperimentalWarning: The configuration may change in the future.\n",
      "  driver.verify_connectivity()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Medicine1 is Done!\n",
      "Medicine2 is Done!\n",
      "Medicine3 is Done!\n",
      "Medicine4 is Done!\n",
      "Medicine5 is Done!\n",
      "Biochemistry, Genetics and Molecular Biology1 is Done!\n",
      "Biochemistry, Genetics and Molecular Biology2 is Done!\n",
      "Biochemistry, Genetics and Molecular Biology3 is Done!\n",
      "Social Sciences1 is Done!\n",
      "Engineering1 is Done!\n",
      "Physics and Astronomy1 is Done!\n",
      "Physics and Astronomy2 is Done!\n",
      "Computer Science1 is Done!\n",
      "Environmental Science1 is Done!\n",
      "Agricultural and Biological Sciences1 is Done!\n",
      "Agricultural and Biological Sciences2 is Done!\n",
      "Earth and Planetary Sciences1 is Done!\n",
      "Chemistry1 is Done!\n",
      "Psychology1 is Done!\n",
      "Neuroscience1 is Done!\n",
      "Mathematics1 is Done!\n",
      "Immunology and Microbiology1 is Done!\n",
      "Materials Science1 is Done!\n",
      "Multidisciplinary1 is Done!\n",
      "Arts and Humanities1 is Done!\n",
      "Chemical Engineering1 is Done!\n",
      "Pharmacology, Toxicology and Pharmaceutics1 is Done!\n",
      "Business, Management and Accounting1 is Done!\n",
      "Energy1 is Done!\n",
      "Nursing1 is Done!\n",
      "Economics, Econometrics and Finance1 is Done!\n",
      "Health Professions1 is Done!\n",
      "Decision Sciences1 is Done!\n",
      "Veterinary1 is Done!\n",
      "Dentistry1 is Done!\n"
     ]
    }
   ],
   "source": [
    "# Execution\n",
    "# 13, \n",
    "\n",
    "with GraphDatabase.driver(uri, auth=(user, password)) as driver:\n",
    "    driver.verify_connectivity()\n",
    "    with driver.session(database=\"neo4j\") as session:\n",
    "        # Create nodes & edges\n",
    "        # Traversing csv files\n",
    "        for subject in file_subject:\n",
    "            for n in range(1, 13):\n",
    "                filename = subject + str(n)\n",
    "                path = \"/Users/jennifer/scopus_data/year2014/\"+filename+\".csv\"\n",
    "                if os.path.exists(path):\n",
    "                    # Skipping bad lines (very rare occurrence): + \", on_bad_lines = 'skip'\"\n",
    "                    df = pd.read_csv(path)\n",
    "            \n",
    "                    session.execute_write(neo4j_scopus.neo4j_create_people, df, subject) \n",
    "                    session.execute_write(neo4j_scopus.neo4j_create_publication, df, subject)\n",
    "                    session.execute_write(neo4j_scopus.neo4j_create_author_pub_edge, df)\n",
    "                    print (subject+str(n)+\" is Done!\")  \n",
    "                else:\n",
    "                    if n == 1:\n",
    "                        print(\"Cannot find file: \"+filename)\n",
    "                    break    \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "session.close()\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "78d178f77aa2234760f0004d20edde8d97d9d6aae15b3a03389e75a39bac3a7d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
