{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic Python Flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "# current notebook is inside ./data/python_example1/\n",
    "NOTEBOOK_DIR = Path().resolve()\n",
    "PROJECT_ROOT = NOTEBOOK_DIR.parent.parent     # go up two levels to project root\n",
    "\n",
    "sys.path.append(str(PROJECT_ROOT / \"src\"))\n",
    "\n",
    "from instantgrade import Evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ Student notebook generated at: submissions/student_notebook.ipynb\n"
     ]
    }
   ],
   "source": [
    "from instantgrade.utils.io_utils import generate_student_notebook\n",
    "\n",
    "generate_student_notebook(\n",
    "    instructor_path=\"./pandas_practice.ipynb\",\n",
    "    output_path=\"./submissions/student_notebook.ipynb\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[37m23:37:39 | INFO     | Starting evaluation pipeline...\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | Loading instructor solution...\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | Loaded 11 questions.\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | Discovered 7 submissions to grade.\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | Starting Docker-based evaluation pipeline...\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | [1/7] Grading: sample_solutions copy.ipynb\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | [Docker] Starting grading for sample_solutions copy.ipynb\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | [Docker] Building image instantgrade:latest from python:3.11-slim...\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | [Docker] Using pyproject.toml install mode\u001b[0m\n",
      "\u001b[37m23:37:39 | INFO     | [Docker] Build context: /Volumes/MacSSD/Areas/Github_Repositories/evaluator\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "#0 building with \"desktop-linux\" instance using docker driver\n",
      "\n",
      "#1 [internal] load build definition from Dockerfile\n",
      "#1 transferring dockerfile: 445B done\n",
      "#1 WARN: UndefinedVar: Usage of undefined variable '$PYTHONPATH' (line 15)\n",
      "#1 DONE 0.0s\n",
      "\n",
      "#2 [internal] load metadata for docker.io/library/python:3.11-slim\n",
      "#2 DONE 0.1s\n",
      "\n",
      "#3 [internal] load .dockerignore\n",
      "#3 transferring context: 2B done\n",
      "#3 DONE 0.0s\n",
      "\n",
      "#4 [internal] load build context\n",
      "#4 DONE 0.0s\n",
      "\n",
      "#5 [1/6] FROM docker.io/library/python:3.11-slim@sha256:193fdd0bbcb3d2ae612bd6cc3548d2f7c78d65b549fcaa8af75624c47474444d\n",
      "#5 resolve docker.io/library/python:3.11-slim@sha256:193fdd0bbcb3d2ae612bd6cc3548d2f7c78d65b549fcaa8af75624c47474444d done\n",
      "#5 DONE 0.0s\n",
      "\n",
      "#4 [internal] load build context\n",
      "#4 transferring context: 1.73MB 0.3s done\n",
      "#4 DONE 0.4s\n",
      "\n",
      "#6 [2/6] RUN pip install --no-cache-dir nbformat nbclient pandas openpyxl\n",
      "#6 CACHED\n",
      "\n",
      "#7 [3/6] COPY . /app\n",
      "#7 DONE 1.5s\n",
      "\n",
      "#8 [4/6] WORKDIR /app\n",
      "#8 DONE 0.1s\n",
      "\n",
      "#9 [5/6] RUN pip install .\n",
      "#9 0.462 Processing /app\n",
      "#9 0.463   Installing build dependencies: started\n",
      "#9 4.903   Installing build dependencies: finished with status 'done'\n",
      "#9 4.904   Getting requirements to build wheel: started\n",
      "#9 5.009   Getting requirements to build wheel: finished with status 'done'\n",
      "#9 5.009   Preparing metadata (pyproject.toml): started\n",
      "#9 5.084   Preparing metadata (pyproject.toml): finished with status 'done'\n",
      "#9 5.088 Requirement already satisfied: openpyxl>=3.0.0 in /usr/local/lib/python3.11/site-packages (from instantgrade==0.1.4) (3.1.5)\n",
      "#9 5.092 Requirement already satisfied: pandas>=1.0.0 in /usr/local/lib/python3.11/site-packages (from instantgrade==0.1.4) (2.3.3)\n",
      "#9 5.093 Requirement already satisfied: nbformat>=5.0.0 in /usr/local/lib/python3.11/site-packages (from instantgrade==0.1.4) (5.10.4)\n",
      "#9 5.093 Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.11/site-packages (from instantgrade==0.1.4) (0.10.2)\n",
      "#9 5.341 Collecting click>=7.0 (from instantgrade==0.1.4)\n",
      "#9 5.544   Downloading click-8.3.1-py3-none-any.whl.metadata (2.6 kB)\n",
      "#9 5.566 Requirement already satisfied: jupyter-client>=6.1.12 in /usr/local/lib/python3.11/site-packages (from nbclient>=0.5.0->instantgrade==0.1.4) (8.6.3)\n",
      "#9 5.566 Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /usr/local/lib/python3.11/site-packages (from nbclient>=0.5.0->instantgrade==0.1.4) (5.9.1)\n",
      "#9 5.567 Requirement already satisfied: traitlets>=5.4 in /usr/local/lib/python3.11/site-packages (from nbclient>=0.5.0->instantgrade==0.1.4) (5.14.3)\n",
      "#9 5.571 Requirement already satisfied: fastjsonschema>=2.15 in /usr/local/lib/python3.11/site-packages (from nbformat>=5.0.0->instantgrade==0.1.4) (2.21.2)\n",
      "#9 5.572 Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.11/site-packages (from nbformat>=5.0.0->instantgrade==0.1.4) (4.25.1)\n",
      "#9 5.573 Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.11/site-packages (from openpyxl>=3.0.0->instantgrade==0.1.4) (2.0.0)\n",
      "#9 5.603 Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/site-packages (from pandas>=1.0.0->instantgrade==0.1.4) (2.3.5)\n",
      "#9 5.604 Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/site-packages (from pandas>=1.0.0->instantgrade==0.1.4) (2.9.0.post0)\n",
      "#9 5.604 Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/site-packages (from pandas>=1.0.0->instantgrade==0.1.4) (2025.2)\n",
      "#9 5.605 Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/site-packages (from pandas>=1.0.0->instantgrade==0.1.4) (2025.2)\n",
      "#9 5.614 Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.11/site-packages (from jsonschema>=2.6->nbformat>=5.0.0->instantgrade==0.1.4) (25.4.0)\n",
      "#9 5.614 Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/site-packages (from jsonschema>=2.6->nbformat>=5.0.0->instantgrade==0.1.4) (2025.9.1)\n",
      "#9 5.615 Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/site-packages (from jsonschema>=2.6->nbformat>=5.0.0->instantgrade==0.1.4) (0.37.0)\n",
      "#9 5.615 Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/site-packages (from jsonschema>=2.6->nbformat>=5.0.0->instantgrade==0.1.4) (0.30.0)\n",
      "#9 5.621 Requirement already satisfied: pyzmq>=23.0 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=6.1.12->nbclient>=0.5.0->instantgrade==0.1.4) (27.1.0)\n",
      "#9 5.621 Requirement already satisfied: tornado>=6.2 in /usr/local/lib/python3.11/site-packages (from jupyter-client>=6.1.12->nbclient>=0.5.0->instantgrade==0.1.4) (6.5.2)\n",
      "#9 5.625 Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.11/site-packages (from jupyter-core!=5.0.*,>=4.12->nbclient>=0.5.0->instantgrade==0.1.4) (4.5.1)\n",
      "#9 5.628 Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas>=1.0.0->instantgrade==0.1.4) (1.17.0)\n",
      "#9 5.640 Requirement already satisfied: typing-extensions>=4.4.0 in /usr/local/lib/python3.11/site-packages (from referencing>=0.28.4->jsonschema>=2.6->nbformat>=5.0.0->instantgrade==0.1.4) (4.15.0)\n",
      "#9 5.711 Downloading click-8.3.1-py3-none-any.whl (108 kB)\n",
      "#9 6.033    ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 108.3/108.3 kB 336.5 kB/s eta 0:00:00\n",
      "#9 6.036 Building wheels for collected packages: instantgrade\n",
      "#9 6.038   Building wheel for instantgrade (pyproject.toml): started\n",
      "#9 6.142   Building wheel for instantgrade (pyproject.toml): finished with status 'done'\n",
      "#9 6.142   Created wheel for instantgrade: filename=instantgrade-0.1.4-py3-none-any.whl size=38845 sha256=861c08bcc95cc6dd613c19f66afb38caa841e61d75f04137095d614c9882a2ec\n",
      "#9 6.142   Stored in directory: /tmp/pip-ephem-wheel-cache-ju8m0z7a/wheels/57/0f/98/bb57b2b57b95807699b822a35c022f139d38a02c27922f27ce\n",
      "#9 6.143 Successfully built instantgrade\n",
      "#9 6.211 Installing collected packages: click, instantgrade\n",
      "#9 6.250 Successfully installed click-8.3.1 instantgrade-0.1.4\n",
      "#9 6.250 WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
      "#9 6.252 \n",
      "#9 6.252 [notice] A new release of pip is available: 24.0 -> 25.3\n",
      "#9 6.252 [notice] To update, run: pip install --upgrade pip\n",
      "#9 DONE 6.3s\n",
      "\n",
      "#10 [6/6] WORKDIR /workspace\n",
      "#10 DONE 0.0s\n",
      "\n",
      "#11 exporting to image\n",
      "#11 exporting layers\n",
      "#11 exporting layers 3.3s done\n",
      "#11 exporting manifest sha256:c82f5aabec1aae462d670de0feed65e4ba6d5c146ada70601c9d4c3b62a5bdd7\n",
      "#11 exporting manifest sha256:c82f5aabec1aae462d670de0feed65e4ba6d5c146ada70601c9d4c3b62a5bdd7 done\n",
      "#11 exporting config sha256:4f16de97437107ff09ecf84ebbdbcf0dddae712212357cc837841013b5ec19ea done\n",
      "#11 exporting attestation manifest sha256:46448d6d7a151b82680668ba9b7961ef9f2938ce2fdc2e94d4cdf1432f2c2123 done\n",
      "#11 exporting manifest list sha256:0cf168adc855fd2b066aa388ca842b891a67a8f15a9027c0d81f621e72739246 done\n",
      "#11 naming to docker.io/library/instantgrade:latest done\n",
      "#11 unpacking to docker.io/library/instantgrade:latest\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[37m23:37:53 | INFO     | [Docker] Built image instantgrade:latest successfully.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "#11 unpacking to docker.io/library/instantgrade:latest 1.4s done\n",
      "#11 DONE 4.7s\n",
      "\n",
      " \u001b[33m1 warning found (use docker --debug to expand):\n",
      "\u001b[0m - UndefinedVar: Usage of undefined variable '$PYTHONPATH' (line 15)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[37m23:37:54 | INFO     | [sample_solutions copy.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:54 | INFO     | [sample_solutions copy.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:54 | INFO     | [sample_solutions copy.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:54 | INFO     | [sample_solutions copy.ipynb][docker] [grader] Final resolved student identity -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:54 | INFO     | [sample_solutions copy.ipynb][docker] [grader] ❌ Fatal: student identity still matches instructor defaults. Skipping assertions.\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [2/7] Grading: sample_solutions partial.ipynb\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [Docker] Starting grading for sample_solutions partial.ipynb\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Final resolved student identity -> name='partial name', roll='partial roll number'\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: check_number\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: even_numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: square_number\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: check_password\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: larger_number\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: even_or_odd\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: sum_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: squares_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: count_letter\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] Evaluating question: average_of_three\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [sample_solutions partial.ipynb][docker] [grader] results.json written with 24 rows.\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [3/7] Grading: sample_solutions.ipynb\u001b[0m\n",
      "\u001b[37m23:37:55 | INFO     | [Docker] Starting grading for sample_solutions.ipynb\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions.ipynb][docker] [grader] Final resolved student identity -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions.ipynb][docker] [grader] ❌ Fatal: student identity still matches instructor defaults. Skipping assertions.\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [4/7] Grading: sample_solutions_1.ipynb\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [Docker] Starting grading for sample_solutions_1.ipynb\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Final resolved student identity -> name='Test 1', roll='Test 1 roll number'\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: check_number\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: even_numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: square_number\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: check_password\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: larger_number\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: even_or_odd\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: sum_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: squares_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: count_letter\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] Evaluating question: average_of_three\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [sample_solutions_1.ipynb][docker] [grader] results.json written with 24 rows.\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [5/7] Grading: sample_solutions_2.ipynb\u001b[0m\n",
      "\u001b[37m23:37:56 | INFO     | [Docker] Starting grading for sample_solutions_2.ipynb\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Final resolved student identity -> name='Test 2', roll='Test 2 roll number'\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: check_number\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: even_numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: square_number\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: check_password\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: larger_number\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: even_or_odd\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: sum_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: squares_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: count_letter\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] Evaluating question: average_of_three\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [sample_solutions_2.ipynb][docker] [grader] results.json written with 24 rows.\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [6/7] Grading: student_notebook copy.ipynb\u001b[0m\n",
      "\u001b[37m23:37:57 | INFO     | [Docker] Starting grading for student_notebook copy.ipynb\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Final resolved student identity -> name='zero', roll='zero roll number'\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: check_number\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: even_numbers_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: square_number\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: check_password\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: larger_number\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: even_or_odd\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: sum_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: squares_1_to_n\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: count_letter\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] Evaluating question: average_of_three\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook copy.ipynb][docker] [grader] results.json written with 24 rows.\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [7/7] Grading: student_notebook.ipynb\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [Docker] Starting grading for student_notebook.ipynb\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook.ipynb][docker] [grader] Starting grading...\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook.ipynb][docker] [grader] Instructor defaults -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook.ipynb][docker] [grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook.ipynb][docker] [grader] Final resolved student identity -> name='student name', roll='student roll number'\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [student_notebook.ipynb][docker] [grader] ❌ Fatal: student identity still matches instructor defaults. Skipping assertions.\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | Execution phase completed successfully.\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | [Reporting] Processed 99 result rows.\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | Report generation complete.\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | Total evaluation completed in 18.81s.\u001b[0m\n",
      "\u001b[37m23:37:58 | INFO     | HTML report generated at: reports/evaluation_report.html\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'reports/evaluation_report.html'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from instantgrade.evaluator import Evaluator\n",
    "\n",
    "eval = Evaluator(\n",
    "    solution_file_path=\"./sample_solutions.ipynb\",\n",
    "    submission_folder_path=\"./submissions\",\n",
    "    use_docker=True,       # or False for local test mode\n",
    "    log_path=\"./logs\",\n",
    "    log_level=\"normal\",\n",
    ")\n",
    "\n",
    "report = eval.run()\n",
    "eval.to_html(\"./reports/evaluation_report.html\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'student_path': PosixPath('submissions/sample_solutions copy.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='student name', roll='student roll number'\\n[grader] ❌ Fatal: student identity still matches instructor defaults. Skipping assertions.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 15.05,\n",
       "   'student_meta': {'name': 'student name',\n",
       "    'roll_number': 'student roll number'}},\n",
       "  'results': [{'question': '_identity_check_',\n",
       "    'assertion': '[missing student identity]',\n",
       "    'status': 'failed',\n",
       "    'error': \"Student notebook missing personalized name/roll_number. Please define:\\n\\nname = 'Your Name'\\nroll_number = 'Your Roll Number'\\n\",\n",
       "    'score': 0,\n",
       "    'description': 'Student did not customize name/roll_number.'}]},\n",
       " {'student_path': PosixPath('submissions/sample_solutions partial.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='partial name', roll='partial roll number'\\n[grader] Evaluating question: check_number\\n[grader] Evaluating question: numbers_1_to_n\\n[grader] Evaluating question: even_numbers_1_to_n\\n[grader] Evaluating question: square_number\\n[grader] Evaluating question: check_password\\n[grader] Evaluating question: larger_number\\n[grader] Evaluating question: even_or_odd\\n[grader] Evaluating question: sum_1_to_n\\n[grader] Evaluating question: squares_1_to_n\\n[grader] Evaluating question: count_letter\\n[grader] Evaluating question: average_of_three\\n[grader] results.json written with 24 rows.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 0.62,\n",
       "   'student_meta': {'name': 'partial name',\n",
       "    'roll_number': 'partial roll number'}},\n",
       "  'results': [{'question': 'check_number',\n",
       "    'assertion': \"assert check_number(5) == 'Positive'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(-3) == 'Negative'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(0) == 'Zero'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'numbers_1_to_n',\n",
       "    'assertion': 'assert numbers_1_to_n(5) == [1, 2, 3, 4, 5]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'Example if n = 5 then\\n**Return:** `[1, 2, 3, 4, 5]`'},\n",
       "   {'question': 'even_numbers_1_to_n',\n",
       "    'assertion': 'assert even_numbers_1_to_n(11) == [2, 4, 6, 8, 10]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'Example if n = 11 then\\n**Return:** `[2, 4, 6, 8, 10]`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(4) == 16',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(0) == 0',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('python') == 'Access Granted'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('PYTHON') == 'Access Denied'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('java') == 'Access Denied'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(4, 9) == 9',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(10, 3) == 10',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(5, 5) == 5',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(2) == 'Even'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(3) == 'Odd'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(5) == 15',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(10) == 55',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'squares_1_to_n',\n",
       "    'assertion': 'assert squares_1_to_n(5) == [1, 4, 9, 16, 25]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'if n is 5\\n**Return:** `[1, 4, 9, 16, 25]`'},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('banana', 'a') == 3\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('apple', 'p') == 2\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('python', 'z') == 0\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert round(average_of_three(4, 8, 10), 2) == 7.33',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nTypeError: type NoneType doesn\\'t define __round__ method\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(3, 3, 3) == 3',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(10, 0, 5) == 5',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'}]},\n",
       " {'student_path': PosixPath('submissions/sample_solutions.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='student name', roll='student roll number'\\n[grader] ❌ Fatal: student identity still matches instructor defaults. Skipping assertions.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 0.65,\n",
       "   'student_meta': {'name': 'student name',\n",
       "    'roll_number': 'student roll number'}},\n",
       "  'results': [{'question': '_identity_check_',\n",
       "    'assertion': '[missing student identity]',\n",
       "    'status': 'failed',\n",
       "    'error': \"Student notebook missing personalized name/roll_number. Please define:\\n\\nname = 'Your Name'\\nroll_number = 'Your Roll Number'\\n\",\n",
       "    'score': 0,\n",
       "    'description': 'Student did not customize name/roll_number.'}]},\n",
       " {'student_path': PosixPath('submissions/sample_solutions_1.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='Test 1', roll='Test 1 roll number'\\n[grader] Evaluating question: check_number\\n[grader] Evaluating question: numbers_1_to_n\\n[grader] Evaluating question: even_numbers_1_to_n\\n[grader] Evaluating question: square_number\\n[grader] Evaluating question: check_password\\n[grader] Evaluating question: larger_number\\n[grader] Evaluating question: even_or_odd\\n[grader] Evaluating question: sum_1_to_n\\n[grader] Evaluating question: squares_1_to_n\\n[grader] Evaluating question: count_letter\\n[grader] Evaluating question: average_of_three\\n[grader] results.json written with 24 rows.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 0.63,\n",
       "   'student_meta': {'name': 'Test 1', 'roll_number': 'Test 1 roll number'}},\n",
       "  'results': [{'question': 'check_number',\n",
       "    'assertion': \"assert check_number(5) == 'Positive'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(-3) == 'Negative'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(0) == 'Zero'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'numbers_1_to_n',\n",
       "    'assertion': 'assert numbers_1_to_n(5) == [1, 2, 3, 4, 5]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'Example if n = 5 then\\n**Return:** `[1, 2, 3, 4, 5]`'},\n",
       "   {'question': 'even_numbers_1_to_n',\n",
       "    'assertion': 'assert even_numbers_1_to_n(11) == [2, 4, 6, 8, 10]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'Example if n = 11 then\\n**Return:** `[2, 4, 6, 8, 10]`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(4) == 16',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(0) == 0',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('python') == 'Access Granted'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('PYTHON') == 'Access Denied'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('java') == 'Access Denied'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(4, 9) == 9',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(10, 3) == 10',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(5, 5) == 5',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(2) == 'Even'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(3) == 'Odd'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(5) == 15',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(10) == 55',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'squares_1_to_n',\n",
       "    'assertion': 'assert squares_1_to_n(5) == [1, 4, 9, 16, 25]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'if n is 5\\n**Return:** `[1, 4, 9, 16, 25]`'},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('banana', 'a') == 3\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('apple', 'p') == 2\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('python', 'z') == 0\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert round(average_of_three(4, 8, 10), 2) == 7.33',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(3, 3, 3) == 3',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(10, 0, 5) == 5',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'}]},\n",
       " {'student_path': PosixPath('submissions/sample_solutions_2.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='Test 2', roll='Test 2 roll number'\\n[grader] Evaluating question: check_number\\n[grader] Evaluating question: numbers_1_to_n\\n[grader] Evaluating question: even_numbers_1_to_n\\n[grader] Evaluating question: square_number\\n[grader] Evaluating question: check_password\\n[grader] Evaluating question: larger_number\\n[grader] Evaluating question: even_or_odd\\n[grader] Evaluating question: sum_1_to_n\\n[grader] Evaluating question: squares_1_to_n\\n[grader] Evaluating question: count_letter\\n[grader] Evaluating question: average_of_three\\n[grader] results.json written with 24 rows.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 0.6,\n",
       "   'student_meta': {'name': 'Test 2', 'roll_number': 'Test 2 roll number'}},\n",
       "  'results': [{'question': 'check_number',\n",
       "    'assertion': \"assert check_number(5) == 'Positive'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(-3) == 'Negative'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(0) == 'Zero'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'numbers_1_to_n',\n",
       "    'assertion': 'assert numbers_1_to_n(5) == [1, 2, 3, 4, 5]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'Example if n = 5 then\\n**Return:** `[1, 2, 3, 4, 5]`'},\n",
       "   {'question': 'even_numbers_1_to_n',\n",
       "    'assertion': 'assert even_numbers_1_to_n(11) == [2, 4, 6, 8, 10]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'Example if n = 11 then\\n**Return:** `[2, 4, 6, 8, 10]`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(4) == 16',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(0) == 0',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('python') == 'Access Granted'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('PYTHON') == 'Access Denied'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('java') == 'Access Denied'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(4, 9) == 9',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(10, 3) == 10',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(5, 5) == 5',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(2) == 'Even'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(3) == 'Odd'\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(5) == 15',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(10) == 55',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'squares_1_to_n',\n",
       "    'assertion': 'assert squares_1_to_n(5) == [1, 4, 9, 16, 25]',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': 'if n is 5\\n**Return:** `[1, 4, 9, 16, 25]`'},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('banana', 'a') == 3\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('apple', 'p') == 2\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('python', 'z') == 0\",\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert round(average_of_three(4, 8, 10), 2) == 7.33',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(3, 3, 3) == 3',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(10, 0, 5) == 5',\n",
       "    'status': 'passed',\n",
       "    'error': None,\n",
       "    'score': 1,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'}]},\n",
       " {'student_path': PosixPath('submissions/student_notebook copy.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='zero', roll='zero roll number'\\n[grader] Evaluating question: check_number\\n[grader] Evaluating question: numbers_1_to_n\\n[grader] Evaluating question: even_numbers_1_to_n\\n[grader] Evaluating question: square_number\\n[grader] Evaluating question: check_password\\n[grader] Evaluating question: larger_number\\n[grader] Evaluating question: even_or_odd\\n[grader] Evaluating question: sum_1_to_n\\n[grader] Evaluating question: squares_1_to_n\\n[grader] Evaluating question: count_letter\\n[grader] Evaluating question: average_of_three\\n[grader] results.json written with 24 rows.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 0.6,\n",
       "   'student_meta': {'name': 'zero', 'roll_number': 'zero roll number'}},\n",
       "  'results': [{'question': 'check_number',\n",
       "    'assertion': \"assert check_number(5) == 'Positive'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(-3) == 'Negative'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'check_number',\n",
       "    'assertion': \"assert check_number(0) == 'Zero'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Return:** A string such as `'Positive'`, `'Negative'`, or `'Zero'`\"},\n",
       "   {'question': 'numbers_1_to_n',\n",
       "    'assertion': 'assert numbers_1_to_n(5) == [1, 2, 3, 4, 5]',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': 'Example if n = 5 then\\n**Return:** `[1, 2, 3, 4, 5]`'},\n",
       "   {'question': 'even_numbers_1_to_n',\n",
       "    'assertion': 'assert even_numbers_1_to_n(11) == [2, 4, 6, 8, 10]',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': 'Example if n = 11 then\\n**Return:** `[2, 4, 6, 8, 10]`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(4) == 16',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'square_number',\n",
       "    'assertion': 'assert square_number(0) == 0',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** For input `4`, return `16`'},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('python') == 'Access Granted'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('PYTHON') == 'Access Denied'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'check_password',\n",
       "    'assertion': \"assert check_password('java') == 'Access Denied'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `'java'` → `'Access Denied'`; Input `'python'` → `'Access Granted'`\"},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(4, 9) == 9',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(10, 3) == 10',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'larger_number',\n",
       "    'assertion': 'assert larger_number(5, 5) == 5',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 9)` → `9`  \\n**Return:** The larger number.'},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(2) == 'Even'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'even_or_odd',\n",
       "    'assertion': \"assert even_or_odd(3) == 'Odd'\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `4` → `'Even'`\"},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(5) == 15',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'sum_1_to_n',\n",
       "    'assertion': 'assert sum_1_to_n(10) == 55',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `5` → `15`\\nbecause 1+2'},\n",
       "   {'question': 'squares_1_to_n',\n",
       "    'assertion': 'assert squares_1_to_n(5) == [1, 4, 9, 16, 25]',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': 'if n is 5\\n**Return:** `[1, 4, 9, 16, 25]`'},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('banana', 'a') == 3\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('apple', 'p') == 2\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'count_letter',\n",
       "    'assertion': \"assert count_letter('python', 'z') == 0\",\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': \"**Example:** Input `('banana', 'a')`\\n**Return:** `3`\"},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert round(average_of_three(4, 8, 10), 2) == 7.33',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nTypeError: type NoneType doesn\\'t define __round__ method\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(3, 3, 3) == 3',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'},\n",
       "   {'question': 'average_of_three',\n",
       "    'assertion': 'assert average_of_three(10, 0, 5) == 5',\n",
       "    'status': 'failed',\n",
       "    'error': 'Traceback (most recent call last):\\n  File \"/app/src/instantgrade/comparison/comparison_service.py\", line 44, in run_assertions\\n    exec(compile(code, \"<assertion>\", \"exec\"), ns)\\n  File \"<assertion>\", line 1, in <module>\\nAssertionError\\n',\n",
       "    'score': 0,\n",
       "    'description': '**Example:** Input `(4, 8, 10)` → `7.3333...`  \\n**Return:** The arithmetic mean of the three numbers.'}]},\n",
       " {'student_path': PosixPath('submissions/student_notebook.ipynb'),\n",
       "  'execution': {'success': True,\n",
       "   'errors': [],\n",
       "   'docker_stdout': \"[grader] Starting grading...\\n[grader] Instructor defaults -> name='student name', roll='student roll number'\\n[grader] Namespace after execution: ['average_of_three', 'check_number', 'check_password', 'count_letter', 'even_numbers_1_to_n', 'even_or_odd', 'larger_number', 'name', 'numbers_1_to_n', 'roll_number', 'square_number', 'squares_1_to_n', 'sum_1_to_n']\\n[grader] Final resolved student identity -> name='student name', roll='student roll number'\\n[grader] ❌ Fatal: student identity still matches instructor defaults. Skipping assertions.\",\n",
       "   'docker_stderr': '',\n",
       "   'elapsed': 0.64,\n",
       "   'student_meta': {'name': 'student name',\n",
       "    'roll_number': 'student roll number'}},\n",
       "  'results': [{'question': '_identity_check_',\n",
       "    'assertion': '[missing student identity]',\n",
       "    'status': 'failed',\n",
       "    'error': \"Student notebook missing personalized name/roll_number. Please define:\\n\\nname = 'Your Name'\\nroll_number = 'Your Roll Number'\\n\",\n",
       "    'score': 0,\n",
       "    'description': 'Student did not customize name/roll_number.'}]}]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report.executed_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'evaluator'",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mModuleNotFoundError\u001b[39m                       Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[5]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mevaluator\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mpath_utils\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m filename_safe_timestamp_format\n\u001b[32m      3\u001b[39m result.to_html(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m./report/report_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfilename_safe_timestamp_format()\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.html\u001b[39m\u001b[33m\"\u001b[39m)\n",
      "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'evaluator'"
     ]
    }
   ],
   "source": [
    "from evaluator.utils.path_utils import filename_safe_timestamp_format\n",
    "\n",
    "result.to_html(f\"./report/report_{filename_safe_timestamp_format()}.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "x2JVLvRAu_lX"
   },
   "source": [
    "# EOF"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": ".venv (3.13.7)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
