{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo: Load a model from RobustBench\n",
    "\n",
    "Load a pre-trained model from RobustBench and evaluate the model's robustness\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: timm in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (0.8.19.dev0)\n",
      "Requirement already satisfied: torch>=1.7 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm) (2.1.0)\n",
      "Requirement already satisfied: torchvision in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm) (0.16.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm) (6.0.1)\n",
      "Requirement already satisfied: huggingface-hub in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm) (0.18.0)\n",
      "Requirement already satisfied: safetensors in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm) (0.4.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch>=1.7->timm) (3.12.4)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch>=1.7->timm) (4.8.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch>=1.7->timm) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch>=1.7->timm) (3.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch>=1.7->timm) (3.1.2)\n",
      "Requirement already satisfied: fsspec in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch>=1.7->timm) (2023.9.2)\n",
      "Requirement already satisfied: requests in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from huggingface-hub->timm) (2.25.1)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from huggingface-hub->timm) (4.66.1)\n",
      "Requirement already satisfied: packaging>=20.9 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from huggingface-hub->timm) (23.2)\n",
      "Requirement already satisfied: numpy in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torchvision->timm) (1.26.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torchvision->timm) (10.0.1)\n",
      "Requirement already satisfied: colorama in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tqdm>=4.42.1->huggingface-hub->timm) (0.4.6)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from jinja2->torch>=1.7->timm) (2.1.3)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->huggingface-hub->timm) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->huggingface-hub->timm) (2.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->huggingface-hub->timm) (1.26.17)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->huggingface-hub->timm) (2023.7.22)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from sympy->torch>=1.7->timm) (1.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "detectors 0.1.10 requires timm==0.8.19.dev0, but you have timm 0.6.13 which is incompatible.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: adversarial-insight-ml in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (0.2.0)\n",
      "Requirement already satisfied: adversarial-robustness-toolbox in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-insight-ml) (1.16.0)\n",
      "Requirement already satisfied: torch in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-insight-ml) (2.1.0)\n",
      "Requirement already satisfied: detectors in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-insight-ml) (0.1.10)\n",
      "Requirement already satisfied: lightning in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-insight-ml) (2.1.0)\n",
      "Requirement already satisfied: torchmetrics in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-insight-ml) (1.2.0)\n",
      "Requirement already satisfied: numpy>=1.18.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-robustness-toolbox->adversarial-insight-ml) (1.26.0)\n",
      "Requirement already satisfied: scipy>=1.4.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-robustness-toolbox->adversarial-insight-ml) (1.11.3)\n",
      "Requirement already satisfied: scikit-learn<1.2.0,>=0.22.2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-robustness-toolbox->adversarial-insight-ml) (1.1.3)\n",
      "Requirement already satisfied: six in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-robustness-toolbox->adversarial-insight-ml) (1.16.0)\n",
      "Requirement already satisfied: setuptools in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-robustness-toolbox->adversarial-insight-ml) (68.2.2)\n",
      "Requirement already satisfied: tqdm in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from adversarial-robustness-toolbox->adversarial-insight-ml) (4.66.1)\n",
      "Requirement already satisfied: Pillow in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (10.0.1)\n",
      "Requirement already satisfied: optuna in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (3.3.0)\n",
      "Requirement already satisfied: scikit-image in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (0.22.0)\n",
      "Collecting timm==0.8.19.dev0 (from detectors->adversarial-insight-ml)\n",
      "  Using cached timm-0.8.19.dev0-py3-none-any.whl (2.2 MB)\n",
      "Requirement already satisfied: torchvision in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (0.16.0)\n",
      "Requirement already satisfied: accelerate in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (0.23.0)\n",
      "Requirement already satisfied: psutil in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (5.9.5)\n",
      "Requirement already satisfied: pandas in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (1.3.5)\n",
      "Requirement already satisfied: wilds in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (2.0.0)\n",
      "Requirement already satisfied: faiss-cpu in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (1.7.4)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from detectors->adversarial-insight-ml) (3.8.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm==0.8.19.dev0->detectors->adversarial-insight-ml) (6.0.1)\n",
      "Requirement already satisfied: huggingface-hub in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm==0.8.19.dev0->detectors->adversarial-insight-ml) (0.18.0)\n",
      "Requirement already satisfied: safetensors in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from timm==0.8.19.dev0->detectors->adversarial-insight-ml) (0.4.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch->adversarial-insight-ml) (3.12.4)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch->adversarial-insight-ml) (4.8.0)\n",
      "Requirement already satisfied: sympy in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch->adversarial-insight-ml) (1.12)\n",
      "Requirement already satisfied: networkx in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch->adversarial-insight-ml) (3.1)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch->adversarial-insight-ml) (3.1.2)\n",
      "Requirement already satisfied: fsspec in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from torch->adversarial-insight-ml) (2023.9.2)\n",
      "Requirement already satisfied: lightning-utilities<2.0,>=0.8.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from lightning->adversarial-insight-ml) (0.9.0)\n",
      "Requirement already satisfied: packaging<25.0,>=20.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from lightning->adversarial-insight-ml) (23.2)\n",
      "Requirement already satisfied: pytorch-lightning in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from lightning->adversarial-insight-ml) (2.1.0)\n",
      "Requirement already satisfied: requests in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from fsspec->torch->adversarial-insight-ml) (2.25.1)\n",
      "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from fsspec->torch->adversarial-insight-ml) (3.8.6)\n",
      "Requirement already satisfied: joblib>=1.0.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from scikit-learn<1.2.0,>=0.22.2->adversarial-robustness-toolbox->adversarial-insight-ml) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from scikit-learn<1.2.0,>=0.22.2->adversarial-robustness-toolbox->adversarial-insight-ml) (3.2.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tqdm->adversarial-robustness-toolbox->adversarial-insight-ml) (0.4.6)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from jinja2->torch->adversarial-insight-ml) (2.1.3)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (4.43.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (1.4.5)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (2.8.2)\n",
      "Requirement already satisfied: importlib-resources>=3.2.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from matplotlib->detectors->adversarial-insight-ml) (6.1.0)\n",
      "Requirement already satisfied: alembic>=1.5.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from optuna->detectors->adversarial-insight-ml) (1.12.0)\n",
      "Requirement already satisfied: cmaes>=0.10.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from optuna->detectors->adversarial-insight-ml) (0.10.0)\n",
      "Requirement already satisfied: colorlog in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from optuna->detectors->adversarial-insight-ml) (6.7.0)\n",
      "Requirement already satisfied: sqlalchemy>=1.3.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from optuna->detectors->adversarial-insight-ml) (2.0.22)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from pandas->detectors->adversarial-insight-ml) (2023.3.post1)\n",
      "Requirement already satisfied: imageio>=2.27 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from scikit-image->detectors->adversarial-insight-ml) (2.31.5)\n",
      "Requirement already satisfied: tifffile>=2022.8.12 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from scikit-image->detectors->adversarial-insight-ml) (2023.9.26)\n",
      "Requirement already satisfied: lazy_loader>=0.3 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from scikit-image->detectors->adversarial-insight-ml) (0.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from sympy->torch->adversarial-insight-ml) (1.3.0)\n",
      "Requirement already satisfied: ogb>=1.2.6 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from wilds->detectors->adversarial-insight-ml) (1.3.6)\n",
      "Requirement already satisfied: outdated>=0.2.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from wilds->detectors->adversarial-insight-ml) (0.2.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (3.3.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec->torch->adversarial-insight-ml) (1.3.1)\n",
      "Requirement already satisfied: Mako in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from alembic>=1.5.0->optuna->detectors->adversarial-insight-ml) (1.2.4)\n",
      "Requirement already satisfied: zipp>=3.1.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from importlib-resources>=3.2.0->matplotlib->detectors->adversarial-insight-ml) (3.17.0)\n",
      "Requirement already satisfied: urllib3>=1.24.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from ogb>=1.2.6->wilds->detectors->adversarial-insight-ml) (1.26.17)\n",
      "Requirement already satisfied: littleutils in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from outdated>=0.2.0->wilds->detectors->adversarial-insight-ml) (0.2.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from sqlalchemy>=1.3.0->optuna->detectors->adversarial-insight-ml) (3.0.0)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->fsspec->torch->adversarial-insight-ml) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->fsspec->torch->adversarial-insight-ml) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests->fsspec->torch->adversarial-insight-ml) (2023.7.22)\n",
      "Installing collected packages: timm\n",
      "  Attempting uninstall: timm\n",
      "    Found existing installation: timm 0.6.13\n",
      "    Uninstalling timm-0.6.13:\n",
      "      Successfully uninstalled timm-0.6.13\n",
      "Successfully installed timm-0.8.19.dev0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "robustbench 1.1 requires timm~=0.6.7, but you have timm 0.8.19.dev0 which is incompatible.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorboard in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (2.14.1)Note: you may need to restart the kernel to use updated packages.\n",
      "\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (2.0.0)\n",
      "Requirement already satisfied: grpcio>=1.48.2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (1.59.0)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (2.23.3)\n",
      "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (1.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (3.5)\n",
      "Requirement already satisfied: numpy>=1.12.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (1.26.0)\n",
      "Requirement already satisfied: protobuf>=3.19.6 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (4.24.4)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (2.25.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (68.2.2)\n",
      "Requirement already satisfied: six>1.9 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (1.16.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (0.7.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from tensorboard) (3.0.0)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard) (5.3.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard) (0.3.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard) (4.9)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from markdown>=2.6.8->tensorboard) (6.8.0)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard) (2.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard) (1.26.17)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard) (2023.7.22)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard) (2.1.3)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard) (3.17.0)\n",
      "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard) (0.5.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard) (3.2.2)\n"
     ]
    }
   ],
   "source": [
    "%pip install timm\n",
    "%pip install -q git+https://github.com/RobustBench/robustbench.git@2d630bc9e8d1cf50d46a4dda65fd36850e3ef769\n",
    "%pip install adversarial-insight-ml\n",
    "%pip install tensorboard\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AIML package (0.2.0) is being initialized.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import aiml\n",
    "from aiml.evaluation.evaluate import evaluate\n",
    "from robustbench.data import load_cifar10\n",
    "from robustbench.utils import load_model\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset\n",
    "import torchvision as tv\n",
    "import torchvision.transforms as T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data\\cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 170498071/170498071 [00:33<00:00, 5051738.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data\\cifar-10-python.tar.gz to ./data\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# x_test, y_test = load_cifar10(n_examples=50)\n",
    "# x_test=torch.Tensor(x_test)\n",
    "# y_test=torch.Tensor(y_test)\n",
    "# dataset = TensorDataset(x_test, y_test)\n",
    "\n",
    "\n",
    "transform = T.Compose([T.ToTensor(), T.Normalize([0.4914, 0.4822, 0.4465], [0.2470, 0.2435, 0.2616])])\n",
    "test_dataset = tv.datasets.CIFAR10('./data', download=True, train=False, transform=transform)\n",
    "train_dataset = tv.datasets.CIFAR10('./data', download=True, train=True, transform=transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download started: path=models\\cifar10\\Linf\\Carmon2019Unlabeled.pt (gdrive_id=15tUx-gkZMYx7BfEOw1GY5OKC-jECIsPQ)\n",
      "Download finished: path=models\\cifar10\\Linf\\Carmon2019Unlabeled.pt (gdrive_id=15tUx-gkZMYx7BfEOw1GY5OKC-jECIsPQ)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model = load_model(model_name='Carmon2019Unlabeled')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating the surrogate model. This may take a long time.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "Missing logger folder: logs\\surrogate\n",
      "\n",
      "  | Name       | Type               | Params\n",
      "--------------------------------------------------\n",
      "0 | oracle     | LogSoftmaxModule   | 38.1 M\n",
      "1 | substitute | VGG                | 134 M \n",
      "2 | loss_fn    | KLDivLoss          | 0     \n",
      "3 | accuracy   | MulticlassAccuracy | 0     \n",
      "--------------------------------------------------\n",
      "134 M     Trainable params\n",
      "38.1 M    Non-trainable params\n",
      "172 M     Total params\n",
      "689.685   Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:436: Consider setting `persistent_workers=True` in 'val_dataloader' to speed up the dataloader worker initialization.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                           \r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages\\pytorch_lightning\\trainer\\connectors\\data_connector.py:436: Consider setting `persistent_workers=True` in 'train_dataloader' to speed up the dataloader worker initialization.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 3/782 [00:32<2:18:32,  0.09it/s, v_num=0]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\takuy\\miniconda3\\envs\\demo_test\\lib\\site-packages\\pytorch_lightning\\trainer\\call.py:54: Detected KeyboardInterrupt, attempting graceful shutdown...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Surrogate model created successfully.\n"
     ]
    }
   ],
   "source": [
    "evaluate(model,input_train_data=train_dataset,input_test_data=test_dataset)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lightning_test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
