{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c63176ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pyspark\n",
    "\n",
    "import pyscisci.all as pyscisci\n",
    "import time\n",
    "\n",
    "path2mag = '/users/hgt6rn/Documents/DataSets/MAG'\n",
    "\n",
    "mymag = pyscisci.MAG(path2mag, database_extension='hdf', show_progress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "37b48533",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4279539,)\n"
     ]
    }
   ],
   "source": [
    "pub2year = mymag.pub2year\n",
    "pub2000 = np.sort([pid for pid,y in pub2year.items() if y==2000])\n",
    "print(pub2000.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "591e910a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de4dd3bfe7b7482a9aecbfa36b105fdf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pub2ref = mymag.load_references(filter_dict={'CitedPublicationId':pub2000})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4d367ebc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37.078683853149414\n"
     ]
    }
   ],
   "source": [
    "stime = time.time()\n",
    "\n",
    "ncites = pub2ref.groupby('CitedPublicationId')['CitingPublicationId'].nunique()\n",
    "\n",
    "etime = time.time()\n",
    "print(etime-stime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edfad506",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/09 14:49:42 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/pyspark/sql/pandas/conversion.py:474: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n",
      "  for column, series in pdf.iteritems():\n",
      "/opt/homebrew/lib/python3.10/site-packages/pyspark/sql/pandas/conversion.py:486: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n",
      "  for column, series in pdf.iteritems():\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create df: 402.3670480251312\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import col, countDistinct\n",
    "\n",
    "spark = SparkSession.builder.config(\"spark.driver.memory\", \"50g\").getOrCreate()\n",
    "\n",
    "stime = time.time()\n",
    "df = spark.createDataFrame(pub2ref)\n",
    "print(\"Create df:\", time.time()-stime)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "854dc342",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/09 14:56:26 WARN TaskSetManager: Stage 0 contains a task of very large size (66074 KiB). The maximum recommended task size is 1000 KiB.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:====================================================>     (9 + 1) / 10]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/12/09 14:57:27 ERROR PythonRunner: Python worker exited unexpectedly (crashed)\n",
      "java.net.SocketException: Connection reset\n",
      "\tat java.base/java.net.SocketInputStream.read(SocketInputStream.java:186)\n",
      "\tat java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)\n",
      "\tat java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)\n",
      "\tat java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:292)\n",
      "\tat java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:351)\n",
      "\tat java.base/java.io.DataInputStream.readFully(DataInputStream.java:200)\n",
      "\tat java.base/java.io.DataInputStream.readFully(DataInputStream.java:170)\n",
      "\tat org.apache.spark.api.python.PythonRunner$$anon$3.read(PythonRunner.scala:758)\n",
      "\tat org.apache.spark.api.python.PythonRunner$$anon$3.read(PythonRunner.scala:747)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:512)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:491)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.hashAgg_doAggregateWithKeys_0$(Unknown Source)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:760)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\n",
      "\tat org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:140)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:136)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:829)\n",
      "22/12/09 14:57:27 ERROR PythonRunner: This may have been caused by a prior exception:\n",
      "java.net.SocketException: Broken pipe (Write failed)\n",
      "\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n",
      "\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n",
      "\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n",
      "\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n",
      "\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n",
      "\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n",
      "\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n",
      "\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n",
      "\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n",
      "\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n",
      "\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n",
      "\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n",
      "22/12/09 14:57:27 ERROR Executor: Exception in task 2.0 in stage 0.0 (TID 2)\n",
      "java.net.SocketException: Broken pipe (Write failed)\n",
      "\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n",
      "\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n",
      "\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n",
      "\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n",
      "\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n",
      "\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n",
      "\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n",
      "\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n",
      "\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n",
      "\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n",
      "\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n",
      "\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n",
      "22/12/09 14:57:27 WARN TaskSetManager: Lost task 2.0 in stage 0.0 (TID 2) (172.25.42.61 executor driver): java.net.SocketException: Broken pipe (Write failed)\n",
      "\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n",
      "\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n",
      "\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n",
      "\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n",
      "\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n",
      "\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n",
      "\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n",
      "\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n",
      "\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n",
      "\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n",
      "\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n",
      "\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n",
      "\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n",
      "\n",
      "22/12/09 14:57:27 ERROR TaskSetManager: Task 2 in stage 0.0 failed 1 times; aborting job\n"
     ]
    },
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling o56.showString.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (172.25.42.61 executor driver): java.net.SocketException: Broken pipe (Write failed)\n\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2672)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2608)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2607)\n\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2607)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1182)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1182)\n\tat scala.Option.foreach(Option.scala:407)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1182)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2860)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2802)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2791)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\nCaused by: java.net.SocketException: Broken pipe (Write failed)\n\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [6], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m stime \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m----> 3\u001b[0m \u001b[43mdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroupby\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mCitedPublicationId\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43magg\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcountDistinct\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mCitingPublicationId\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshow\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCompute Groupby:\u001b[39m\u001b[38;5;124m\"\u001b[39m, time\u001b[38;5;241m.\u001b[39mtime()\u001b[38;5;241m-\u001b[39mstime)\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/pyspark/sql/dataframe.py:606\u001b[0m, in \u001b[0;36mDataFrame.show\u001b[0;34m(self, n, truncate, vertical)\u001b[0m\n\u001b[1;32m    603\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mParameter \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mvertical\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m must be a bool\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    605\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(truncate, \u001b[38;5;28mbool\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m truncate:\n\u001b[0;32m--> 606\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshowString\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m20\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvertical\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    607\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    608\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/py4j/java_gateway.py:1321\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1315\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m   1316\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m   1317\u001b[0m     args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m   1318\u001b[0m     proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[1;32m   1320\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client\u001b[38;5;241m.\u001b[39msend_command(command)\n\u001b[0;32m-> 1321\u001b[0m return_value \u001b[38;5;241m=\u001b[39m \u001b[43mget_return_value\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1322\u001b[0m \u001b[43m    \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtarget_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1324\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n\u001b[1;32m   1325\u001b[0m     temp_arg\u001b[38;5;241m.\u001b[39m_detach()\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/pyspark/sql/utils.py:190\u001b[0m, in \u001b[0;36mcapture_sql_exception.<locals>.deco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m    188\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdeco\u001b[39m(\u001b[38;5;241m*\u001b[39ma: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[1;32m    189\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 190\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    191\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m Py4JJavaError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    192\u001b[0m         converted \u001b[38;5;241m=\u001b[39m convert_exception(e\u001b[38;5;241m.\u001b[39mjava_exception)\n",
      "File \u001b[0;32m/opt/homebrew/lib/python3.10/site-packages/py4j/protocol.py:326\u001b[0m, in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    324\u001b[0m value \u001b[38;5;241m=\u001b[39m OUTPUT_CONVERTER[\u001b[38;5;28mtype\u001b[39m](answer[\u001b[38;5;241m2\u001b[39m:], gateway_client)\n\u001b[1;32m    325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m answer[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m==\u001b[39m REFERENCE_TYPE:\n\u001b[0;32m--> 326\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m Py4JJavaError(\n\u001b[1;32m    327\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[1;32m    328\u001b[0m         \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name), value)\n\u001b[1;32m    329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    330\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m Py4JError(\n\u001b[1;32m    331\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m. Trace:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{3}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[1;32m    332\u001b[0m         \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name, value))\n",
      "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o56.showString.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 0.0 failed 1 times, most recent failure: Lost task 2.0 in stage 0.0 (TID 2) (172.25.42.61 executor driver): java.net.SocketException: Broken pipe (Write failed)\n\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2672)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2608)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2607)\n\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2607)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1182)\n\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1182)\n\tat scala.Option.foreach(Option.scala:407)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1182)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2860)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2802)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2791)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\nCaused by: java.net.SocketException: Broken pipe (Write failed)\n\tat java.base/java.net.SocketOutputStream.socketWrite0(Native Method)\n\tat java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110)\n\tat java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150)\n\tat java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)\n\tat java.base/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)\n\tat java.base/java.io.DataOutputStream.write(DataOutputStream.java:107)\n\tat java.base/java.io.FilterOutputStream.write(FilterOutputStream.java:108)\n\tat org.apache.spark.api.python.PythonRDD$.write$1(PythonRDD.scala:295)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRDD$.$anonfun$writeIteratorToStream$1$adapted(PythonRDD.scala:307)\n\tat scala.collection.Iterator.foreach(Iterator.scala:943)\n\tat scala.collection.Iterator.foreach$(Iterator.scala:943)\n\tat org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)\n\tat org.apache.spark.api.python.PythonRDD$.writeIteratorToStream(PythonRDD.scala:307)\n\tat org.apache.spark.api.python.PythonRunner$$anon$2.writeIteratorToStream(PythonRunner.scala:732)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.$anonfun$run$1(PythonRunner.scala:438)\n\tat org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2066)\n\tat org.apache.spark.api.python.BasePythonRunner$WriterThread.run(PythonRunner.scala:272)\n"
     ]
    }
   ],
   "source": [
    "stime = time.time()\n",
    "\n",
    "df.groupby('CitedPublicationId').agg(countDistinct('CitingPublicationId')).show()\n",
    "\n",
    "print(\"Compute Groupby:\", time.time()-stime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbe88258",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
