{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "63ce0734-aad1-48c2-948c-f6ad0606132f",
   "metadata": {},
   "source": [
    "## Run a pipeline using dask and threads"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdbd6fb8-f668-4a18-bbb7-f92a3bdf2172",
   "metadata": {},
   "source": [
    "This process will run the entire pipeline described in a configuration file using dask and threads"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "118c14ec-8c74-4341-99b9-4099862de1f7",
   "metadata": {},
   "source": [
    "## Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f00299ae-1140-494b-99be-9fa8867c0ac8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from paidiverpy.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89f47537-0ef2-473e-b7f7-a572cebce86b",
   "metadata": {},
   "source": [
    "## Instantiate the Pipeline class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29c5be66-c950-4ea3-9448-b7618e0ea52a",
   "metadata": {},
   "source": [
    "This pipeline will process the dataset using dask and threads. In this case, as \"n_jobs: -1\", it will use all the processors that I have on my machine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "46f413c1-b9a5-4dc1-a27d-9cba7f9c745c",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline(config_file_path=\"../config_files/config_benthic_dask.yml\", verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "491bd5dd-5462-4905-8a62-7e57e83aacc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
       "<defs>\n",
       "<symbol id=\"arrow-right\" viewBox=\"0 0 330 330\">\n",
       "  <path id=\"XMLID_222_\" d=\"M250.606,154.389l-150-149.996c-5.857-5.858-15.355-5.858-21.213,0.001\n",
       "\tc-5.857,5.858-5.857,15.355,0.001,21.213l139.393,139.39L79.393,304.394c-5.857,5.858-5.857,15.355,0.001,21.213\n",
       "\tC82.322,328.536,86.161,330,90,330s7.678-1.464,10.607-4.394l149.999-150.004c2.814-2.813,4.394-6.628,4.394-10.606\n",
       "\tC255,161.018,253.42,157.202,250.606,154.389z\"/>\n",
       "</defs>\n",
       "</svg>\n",
       "<style>/* CSS stylesheet for displaying Paidiverpy objects in jupyterlab.\n",
       " *\n",
       " */\n",
       "\n",
       ":root {\n",
       "  --ppy-font-color: var(--jp-content-font-color, rgba(0, 0, 0, 1));\n",
       "  --ppy-font-color-brown: var(--jp-content-font-color-brown, brown);\n",
       "  --ppy-font-color-olive: var(--jp-content-font-color-olive, olive);\n",
       "  --ppy-font-color-teal: var(--jp-content-font-color-teal, teal);\n",
       "  --ppy-font-color-blue: var(--jp-content-font-color-blue, blue);\n",
       "  --ppy-font-color-gray: var(--jp-content-font-color-gray, gray);\n",
       "  --ppy-border-color: var(--jp-border-color2, #e0e0e0);\n",
       "  --ppy-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
       "  --ppy-background-color: var(--jp-layout-color, white);\n",
       "  --ppy-background-color2: var(--jp-layout-color3, rgba(212, 212, 212, 0.15));\n",
       "}\n",
       "\n",
       "\n",
       "html[theme=\"dark\"],\n",
       "html[data-theme=\"dark\"],\n",
       "body[data-theme=\"dark\"],\n",
       "body.vscode-dark {\n",
       "  --ppy-font-color: rgba(255, 255, 255, 1);\n",
       "  --ppy-font-color-brown: brown;\n",
       "  --ppy-font-color-olive: olive;\n",
       "  --ppy-font-color-teal: teal;\n",
       "  --ppy-font-color-blue: blue;\n",
       "  --ppy-font-color-gray: gray;\n",
       "  --ppy-border-color: #1f1f1f;\n",
       "  --ppy-disabled-color: #515151;\n",
       "  --ppy-background-color: #111111;\n",
       "}\n",
       "\n",
       ".ppy-background-color {\n",
       "  background-color: var(--ppy-background-color) !important;\n",
       "}\n",
       "\n",
       ".ppy-font-color-brown {\n",
       "  color: var(--ppy-font-color-brown) !important;\n",
       "}\n",
       "\n",
       ".ppy-font-color {\n",
       "  color: var(--ppy-font-color) !important;\n",
       "}\n",
       ".ppy-h2 {\n",
       "  font-size: 20px !important;\n",
       "}\n",
       ".ppy-h3 {\n",
       "  font-size: 13px !important;\n",
       "}\n",
       ".ppy-p {\n",
       "  font-size: 10px;\n",
       "}\n",
       ".ppy-p-error {\n",
       "  color: var(--ppy-font-color-brown);\n",
       "  font-weight: bold;\n",
       "  display: block;\n",
       "}\n",
       "\n",
       ".ppy-text-repr-fallback {\n",
       "  /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
       "  display: none;\n",
       "}\n",
       "\n",
       ".ppy-pipeline-wrap{\n",
       "  display: flex;\n",
       "  flex-wrap: wrap;\n",
       "  align-items: center;\n",
       "}\n",
       "\n",
       ".ppy-pipeline {\n",
       "  float:left;\n",
       "  cursor: pointer;\n",
       "  padding: 10px;\n",
       "  width: max-content;\n",
       "  height: 150px;\n",
       "  margin: 10px;\n",
       "  border: 1px solid #000;\n",
       "  text-align: center;\n",
       "  /* line-height: 80px; */\n",
       "}\n",
       "\n",
       ".ppy-pipeline-clear-fix {\n",
       "  clear: both;\n",
       "}\n",
       "\n",
       ".ppy-pipeline-parameters-all {\n",
       "  padding: 10px;\n",
       "  margin: 10px;\n",
       "}\n",
       "\n",
       ".ppy-images-step-header {\n",
       "  cursor: pointer;\n",
       "  font-weight: bold;\n",
       "  margin-top: 10px;\n",
       "  margin-bottom: 5px;\n",
       "}\n",
       ".ppy-images-metadata {\n",
       "  margin-left: 20px;\n",
       "  margin-bottom: 10px;\n",
       "}\n",
       ".ppy-images {\n",
       "  display: flex;\n",
       "  flex-wrap: wrap;\n",
       "  gap: 10px;\n",
       "  margin-left: 20px;\n",
       "}\n",
       "\n",
       ".ppy-show-more-button, .ppy-hide-button {\n",
       "  margin-left: 20px;\n",
       "  background-color: #007bff;\n",
       "  color: white;\n",
       "  border: none;\n",
       "  padding: 5px 10px;\n",
       "  cursor: pointer;\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-pipeline-arrow {\n",
       "  float:left;\n",
       "  width: 50px;\n",
       "  height: 80px;\n",
       "  margin: 10px;\n",
       "  text-align: center;\n",
       "  line-height: 80px;\n",
       "}\n",
       "\n",
       ".ppy-images-img{\n",
       "  display: block;\n",
       "  max-width: 200px;\n",
       "  margin: 10px;\n",
       "}\n",
       "\n",
       ".ppy-images-toggle-arrow {\n",
       "  cursor: pointer;\n",
       "  margin-left: 5px;\n",
       "}\n",
       "\n",
       ".ppy-images-image-p {\n",
       "  cursor: pointer;\n",
       "}\n",
       "\n",
       ".ppy-arrow-right {\n",
       "  display: inline-block;\n",
       "  vertical-align: middle;\n",
       "  width: 1em;\n",
       "  height: 1.5em !important;\n",
       "  stroke-width: 0;\n",
       "  stroke: currentColor;\n",
       "  fill: currentColor;\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-json-block {\n",
       "  font-family: monospace;\n",
       "  background: var(--ppy-background-color);\n",
       "  padding: 1em;\n",
       "  border-radius: 6px;\n",
       "}\n",
       ".ppy-json-key {\n",
       "  color: var(--ppy-font-color-brown);\n",
       "}\n",
       ".ppy-json-string {\n",
       "  color: var(--ppy-font-color-olive);\n",
       "}\n",
       ".ppy-json-number {\n",
       "  color: var(--ppy-font-color-teal);\n",
       "}\n",
       ".ppy-json-bool {\n",
       "  color: var(--ppy-font-color-blue);\n",
       "}\n",
       ".ppy-json-null {\n",
       "  color: var(--ppy-font-color-gray);\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-pn-title-0 {\n",
       "  font-size: 30px;\n",
       "}\n",
       "\n",
       ".ppy-pn-title-1 {\n",
       "  font-size: 20px;\n",
       "}\n",
       ".ppy-pn-title-2 {\n",
       "  font-size: 17px;\n",
       "}\n",
       ".ppy-pn-title-3 {\n",
       "  font-size: 14px;\n",
       "}\n",
       ".ppy-pn-title-4 {\n",
       "  font-size: 12px;\n",
       "}\n",
       "\n",
       ".ppy-pn-bold {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".ppy-pn-italic {\n",
       "  font-style: italic;\n",
       "}\n",
       ".ppy-pn-description {\n",
       "  font-size: 14px;\n",
       "  margin-bottom: 1px;\n",
       "}\n",
       "\n",
       ".bk-root {\n",
       "  width: 100% !important;\n",
       "  max-width: 100% !important;\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "\n",
       "input, .bk-input, .bk-input-group {\n",
       "  margin-top: 0px;\n",
       "  margin-bottom: 3px;\n",
       "  padding-top: 0px;\n",
       "  padding-bottom: 0px;\n",
       "}\n",
       "\n",
       ".ppy-pn-header-button .bk-btn {\n",
       "  background: none !important;\n",
       "  border: none !important;\n",
       "  font-weight: bold;\n",
       "  font-size: 16px;\n",
       "  text-align: left;\n",
       "  padding-left: 0;\n",
       "  cursor: pointer;\n",
       "  box-shadow: none !important;\n",
       "  color: black;\n",
       "}\n",
       "\n",
       ".ppy-pn-danger-modal{\n",
       "  border: 2px solid red;\n",
       "  border-radius: 10px;\n",
       "  box-shadow: 0px 0px 10px rgba(255, 0, 0, 0.5);\n",
       "}\n",
       "\n",
       ".ppy-pn-config-form{\n",
       "  padding: 10px;\n",
       "  margin-top: 10px;\n",
       "  margin-bottom: 10px;\n",
       "  background-color: var(--ppy-background-color2);\n",
       "  border-radius: 5px;\n",
       "  box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.1);\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-pn-align-right {\n",
       "  display: flex;\n",
       "  justify-content: flex-end;\n",
       "}\n",
       "</style><script>function toggleMetadata(stepIndex, randomId) {\n",
       "  var metadata = document.getElementById(\"ppy-images-metadata-\" + randomId + \"-\" + stepIndex);\n",
       "  var arrow = document.getElementById(\"ppy-images-arrow-\" + randomId + \"-\" + stepIndex);\n",
       "  if (metadata.style.display === \"none\") {\n",
       "      metadata.style.display = \"block\";\n",
       "      arrow.innerHTML = \"▼\";\n",
       "      arrow.classList.remove(\"ppy-font-color\");\n",
       "      arrow.classList.add(\"ppy-font-color-brown\");\n",
       "  } else {\n",
       "      metadata.style.display = \"none\";\n",
       "      arrow.innerHTML = \"►\";\n",
       "      arrow.classList.remove(\"ppy-font-color-brown\");\n",
       "      arrow.classList.add(\"ppy-font-color\");\n",
       "  }\n",
       "}\n",
       "function toggleImage(imageId, randomId) {\n",
       "  var img = document.getElementById(imageId);\n",
       "  var arrow = document.getElementById(\"ppy-images-arrow-\" + randomId + \"-\" + imageId);\n",
       "  if (img.style.display === \"none\") {\n",
       "      img.style.display = \"block\";\n",
       "      arrow.innerHTML = \"▼\";\n",
       "      arrow.classList.remove(\"ppy-font-color\");\n",
       "      arrow.classList.add(\"ppy-font-color-brown\");\n",
       "  } else {\n",
       "      img.style.display = \"none\";\n",
       "      arrow.innerHTML = \"►\";\n",
       "      arrow.classList.remove(\"ppy-font-color-brown\");\n",
       "      arrow.classList.add(\"ppy-font-color\");\n",
       "  }\n",
       "}\n",
       "function showMore(stepIndex) {\n",
       "  var moreImages = document.getElementById(\"ppy-more-images-\" + stepIndex);\n",
       "  var showMoreButton = document.getElementById(\"ppy-show-more-button-\" + stepIndex);\n",
       "  var hideButton = document.getElementById(\"ppy-hide-button-\" + stepIndex);\n",
       "  moreImages.style.display = \"flex\";\n",
       "  showMoreButton.style.display = \"none\";\n",
       "  hideButton.style.display = \"inline-block\";\n",
       "}\n",
       "function hide(stepIndex) {\n",
       "  var moreImages = document.getElementById(\"ppy-more-images-\" + stepIndex);\n",
       "  var showMoreButton = document.getElementById(\"ppy-show-more-button-\" + stepIndex);\n",
       "  var hideButton = document.getElementById(\"ppy-hide-button-\" + stepIndex);\n",
       "  moreImages.style.display = \"none\";\n",
       "  showMoreButton.style.display = \"inline-block\";\n",
       "  hideButton.style.display = \"none\";\n",
       "}\n",
       "\n",
       "\n",
       "function showParameters(id, randomId) {\n",
       "  var currentTarget = document.getElementById('ppy-pipeline-' + randomId + '-' + id);\n",
       "  var square = document.getElementsByClassName('ppy-pipeline-' + randomId);\n",
       "  var allParams = document.getElementsByClassName('ppy-pipeline-parameters-' + randomId);\n",
       "  var selectedParams = document.getElementById('ppy-pipeline-parameters-' + randomId + '-' + id);\n",
       "  var idWasVisible = false;\n",
       "  if (selectedParams) {\n",
       "      var idWasVisible = selectedParams.style.display === 'block';\n",
       "  }\n",
       "  for (var i = 0; i < square.length; i++) {\n",
       "      square[i].classList.remove('ppy-font-color-brown');\n",
       "      square[i].classList.add('ppy-font-color');\n",
       "  }\n",
       "  for (var i = 0; i < allParams.length; i++) {\n",
       "      allParams[i].style.display = 'none';\n",
       "  }\n",
       "  // Show the selected parameter section\n",
       "  if (selectedParams) {\n",
       "      if (idWasVisible) {\n",
       "          selectedParams.style.display = 'none';\n",
       "          currentTarget.classList.remove('ppy-font-color-brown');\n",
       "          currentTarget.classList.add('ppy-font-color');\n",
       "      } else {\n",
       "          selectedParams.style.display = 'block';\n",
       "          currentTarget.classList.remove('ppy-font-color');\n",
       "          currentTarget.classList.add('ppy-font-color-brown');\n",
       "      }\n",
       "  }\n",
       "}\n",
       "\n",
       "hljs.highlightAll();\n",
       "</script><link rel=\"stylesheet\" href=\"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css\"><script src=\"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js\"></script><script>hljs.highlightAll();</script><pre class='ppy-text-repr-fallback'>{\n",
       "    &quot;general&quot;: {\n",
       "        &quot;name&quot;: &quot;raw&quot;,\n",
       "        &quot;step_name&quot;: &quot;open&quot;,\n",
       "        &quot;sample_data&quot;: &quot;benthic_csv&quot;,\n",
       "        &quot;input_path&quot;: &quot;/home/tobfer/.paidiverpy_cache/benthic_csv/images&quot;,\n",
       "        &quot;output_path&quot;: &quot;output&quot;,\n",
       "        &quot;metadata_path&quot;: &quot;/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/metadata_benthic_csv.csv&quot;,\n",
       "        &quot;metadata_type&quot;: &quot;CSV_FILE&quot;,\n",
       "        &quot;image_open_args&quot;: &quot;PNG&quot;,\n",
       "        &quot;append_data_to_metadata&quot;: &quot;/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/appended_metadata_benthic_csv.csv&quot;,\n",
       "        &quot;n_jobs&quot;: -1,\n",
       "        &quot;track_changes&quot;: true,\n",
       "        &quot;sampling&quot;: [\n",
       "            {\n",
       "                &quot;name&quot;: &quot;sampling&quot;,\n",
       "                &quot;step_name&quot;: &quot;sampling&quot;,\n",
       "                &quot;test&quot;: false,\n",
       "                &quot;params&quot;: {\n",
       "                    &quot;value&quot;: 20,\n",
       "                    &quot;raise_error&quot;: false\n",
       "                },\n",
       "                &quot;mode&quot;: &quot;fixed&quot;\n",
       "            }\n",
       "        ]\n",
       "    },\n",
       "    &quot;steps&quot;: [\n",
       "        {\n",
       "            &quot;name&quot;: &quot;datetime&quot;,\n",
       "            &quot;step_name&quot;: &quot;sampling&quot;,\n",
       "            &quot;test&quot;: false,\n",
       "            &quot;params&quot;: {\n",
       "                &quot;min&quot;: &quot;2018-06-11 04:14:00&quot;,\n",
       "                &quot;max&quot;: &quot;2018-06-11 04:20:00&quot;,\n",
       "                &quot;raise_error&quot;: false\n",
       "            },\n",
       "            &quot;mode&quot;: &quot;datetime&quot;\n",
       "        },\n",
       "        {\n",
       "            &quot;name&quot;: &quot;overlapping&quot;,\n",
       "            &quot;step_name&quot;: &quot;sampling&quot;,\n",
       "            &quot;test&quot;: false,\n",
       "            &quot;params&quot;: {\n",
       "                &quot;omega&quot;: 57.0,\n",
       "                &quot;theta&quot;: 40.0,\n",
       "                &quot;threshold&quot;: 0.5,\n",
       "                &quot;camera_distance&quot;: 1.12,\n",
       "                &quot;raise_error&quot;: false\n",
       "            },\n",
       "            &quot;mode&quot;: &quot;overlapping&quot;\n",
       "        },\n",
       "        {\n",
       "            &quot;name&quot;: &quot;colour_alteration&quot;,\n",
       "            &quot;step_name&quot;: &quot;colour&quot;,\n",
       "            &quot;test&quot;: false,\n",
       "            &quot;params&quot;: {\n",
       "                &quot;method&quot;: &quot;white_balance&quot;,\n",
       "                &quot;raise_error&quot;: false\n",
       "            },\n",
       "            &quot;mode&quot;: &quot;colour_alteration&quot;\n",
       "        },\n",
       "        {\n",
       "            &quot;name&quot;: &quot;gaussian_blur&quot;,\n",
       "            &quot;step_name&quot;: &quot;colour&quot;,\n",
       "            &quot;test&quot;: false,\n",
       "            &quot;params&quot;: {\n",
       "                &quot;sigma&quot;: 1.0,\n",
       "                &quot;raise_error&quot;: false\n",
       "            },\n",
       "            &quot;mode&quot;: &quot;gaussian_blur&quot;\n",
       "        },\n",
       "        {\n",
       "            &quot;name&quot;: &quot;sharpen&quot;,\n",
       "            &quot;step_name&quot;: &quot;colour&quot;,\n",
       "            &quot;test&quot;: false,\n",
       "            &quot;params&quot;: {\n",
       "                &quot;alpha&quot;: 1.5,\n",
       "                &quot;beta&quot;: -0.5,\n",
       "                &quot;raise_error&quot;: false\n",
       "            },\n",
       "            &quot;mode&quot;: &quot;sharpen&quot;\n",
       "        },\n",
       "        {\n",
       "            &quot;name&quot;: &quot;contrast&quot;,\n",
       "            &quot;step_name&quot;: &quot;colour&quot;,\n",
       "            &quot;test&quot;: false,\n",
       "            &quot;params&quot;: {\n",
       "                &quot;method&quot;: &quot;clahe&quot;,\n",
       "                &quot;clip_limit&quot;: 0.01,\n",
       "                &quot;gamma_value&quot;: 0.5,\n",
       "                &quot;raise_error&quot;: false\n",
       "            },\n",
       "            &quot;mode&quot;: &quot;contrast&quot;\n",
       "        }\n",
       "    ]\n",
       "}</pre><div><pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">general</span>:\n",
       "  <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"raw\"</span>\n",
       "  <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"open\"</span>\n",
       "  <span class=\"ppy-json-key\">sample_data</span>: <span class=\"ppy-json-string\">\"benthic_csv\"</span>\n",
       "  <span class=\"ppy-json-key\">input_path</span>: <span class=\"ppy-json-string\">\"/home/tobfer/.paidiverpy_cache/benthic_csv/images\"</span>\n",
       "  <span class=\"ppy-json-key\">output_path</span>: <span class=\"ppy-json-string\">\"output\"</span>\n",
       "  <span class=\"ppy-json-key\">metadata_path</span>: <span class=\"ppy-json-string\">\"/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/metadata_benthic_csv.csv\"</span>\n",
       "  <span class=\"ppy-json-key\">metadata_type</span>: <span class=\"ppy-json-string\">\"CSV_FILE\"</span>\n",
       "  <span class=\"ppy-json-key\">image_open_args</span>: <span class=\"ppy-json-string\">\"PNG\"</span>\n",
       "  <span class=\"ppy-json-key\">append_data_to_metadata</span>: <span class=\"ppy-json-string\">\"/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/appended_metadata_benthic_csv.csv\"</span>\n",
       "  <span class=\"ppy-json-key\">n_jobs</span>: <span class=\"ppy-json-number\">-1</span>\n",
       "  <span class=\"ppy-json-key\">track_changes</span>: <span class=\"ppy-json-number\">True</span>\n",
       "  <span class=\"ppy-json-key\">sampling</span>:\n",
       "    - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "      <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "      <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "      <span class=\"ppy-json-key\">params</span>:\n",
       "        <span class=\"ppy-json-key\">value</span>: <span class=\"ppy-json-number\">20</span>\n",
       "        <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "      <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"fixed\"</span>\n",
       "<span class=\"ppy-json-key\">steps</span>:\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"datetime\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">min</span>: <span class=\"ppy-json-string\">\"2018-06-11 04:14:00\"</span>\n",
       "      <span class=\"ppy-json-key\">max</span>: <span class=\"ppy-json-string\">\"2018-06-11 04:20:00\"</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"datetime\"</span>\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"overlapping\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">omega</span>: <span class=\"ppy-json-number\">57.0</span>\n",
       "      <span class=\"ppy-json-key\">theta</span>: <span class=\"ppy-json-number\">40.0</span>\n",
       "      <span class=\"ppy-json-key\">threshold</span>: <span class=\"ppy-json-number\">0.5</span>\n",
       "      <span class=\"ppy-json-key\">camera_distance</span>: <span class=\"ppy-json-number\">1.12</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"overlapping\"</span>\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"colour_alteration\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">method</span>: <span class=\"ppy-json-string\">\"white_balance\"</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"colour_alteration\"</span>\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"gaussian_blur\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">sigma</span>: <span class=\"ppy-json-number\">1.0</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"gaussian_blur\"</span>\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"sharpen\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">alpha</span>: <span class=\"ppy-json-number\">1.5</span>\n",
       "      <span class=\"ppy-json-key\">beta</span>: <span class=\"ppy-json-number\">-0.5</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"sharpen\"</span>\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"contrast\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">method</span>: <span class=\"ppy-json-string\">\"clahe\"</span>\n",
       "      <span class=\"ppy-json-key\">clip_limit</span>: <span class=\"ppy-json-number\">0.01</span>\n",
       "      <span class=\"ppy-json-key\">gamma_value</span>: <span class=\"ppy-json-number\">0.5</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"contrast\"</span></pre></div></div>"
      ],
      "text/plain": [
       "{\n",
       "    \"general\": {\n",
       "        \"name\": \"raw\",\n",
       "        \"step_name\": \"open\",\n",
       "        \"sample_data\": \"benthic_csv\",\n",
       "        \"input_path\": \"/home/tobfer/.paidiverpy_cache/benthic_csv/images\",\n",
       "        \"output_path\": \"output\",\n",
       "        \"metadata_path\": \"/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/metadata_benthic_csv.csv\",\n",
       "        \"metadata_type\": \"CSV_FILE\",\n",
       "        \"image_open_args\": \"PNG\",\n",
       "        \"append_data_to_metadata\": \"/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/appended_metadata_benthic_csv.csv\",\n",
       "        \"n_jobs\": -1,\n",
       "        \"track_changes\": true,\n",
       "        \"sampling\": [\n",
       "            {\n",
       "                \"name\": \"sampling\",\n",
       "                \"step_name\": \"sampling\",\n",
       "                \"test\": false,\n",
       "                \"params\": {\n",
       "                    \"value\": 20,\n",
       "                    \"raise_error\": false\n",
       "                },\n",
       "                \"mode\": \"fixed\"\n",
       "            }\n",
       "        ]\n",
       "    },\n",
       "    \"steps\": [\n",
       "        {\n",
       "            \"name\": \"datetime\",\n",
       "            \"step_name\": \"sampling\",\n",
       "            \"test\": false,\n",
       "            \"params\": {\n",
       "                \"min\": \"2018-06-11 04:14:00\",\n",
       "                \"max\": \"2018-06-11 04:20:00\",\n",
       "                \"raise_error\": false\n",
       "            },\n",
       "            \"mode\": \"datetime\"\n",
       "        },\n",
       "        {\n",
       "            \"name\": \"overlapping\",\n",
       "            \"step_name\": \"sampling\",\n",
       "            \"test\": false,\n",
       "            \"params\": {\n",
       "                \"omega\": 57.0,\n",
       "                \"theta\": 40.0,\n",
       "                \"threshold\": 0.5,\n",
       "                \"camera_distance\": 1.12,\n",
       "                \"raise_error\": false\n",
       "            },\n",
       "            \"mode\": \"overlapping\"\n",
       "        },\n",
       "        {\n",
       "            \"name\": \"colour_alteration\",\n",
       "            \"step_name\": \"colour\",\n",
       "            \"test\": false,\n",
       "            \"params\": {\n",
       "                \"method\": \"white_balance\",\n",
       "                \"raise_error\": false\n",
       "            },\n",
       "            \"mode\": \"colour_alteration\"\n",
       "        },\n",
       "        {\n",
       "            \"name\": \"gaussian_blur\",\n",
       "            \"step_name\": \"colour\",\n",
       "            \"test\": false,\n",
       "            \"params\": {\n",
       "                \"sigma\": 1.0,\n",
       "                \"raise_error\": false\n",
       "            },\n",
       "            \"mode\": \"gaussian_blur\"\n",
       "        },\n",
       "        {\n",
       "            \"name\": \"sharpen\",\n",
       "            \"step_name\": \"colour\",\n",
       "            \"test\": false,\n",
       "            \"params\": {\n",
       "                \"alpha\": 1.5,\n",
       "                \"beta\": -0.5,\n",
       "                \"raise_error\": false\n",
       "            },\n",
       "            \"mode\": \"sharpen\"\n",
       "        },\n",
       "        {\n",
       "            \"name\": \"contrast\",\n",
       "            \"step_name\": \"colour\",\n",
       "            \"test\": false,\n",
       "            \"params\": {\n",
       "                \"method\": \"clahe\",\n",
       "                \"clip_limit\": 0.01,\n",
       "                \"gamma_value\": 0.5,\n",
       "                \"raise_error\": false\n",
       "            },\n",
       "            \"mode\": \"contrast\"\n",
       "        }\n",
       "    ]\n",
       "}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "310d8998-6ac7-4db4-a706-50270b6895ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><svg style=\"position: absolute; width: 0; height: 0; overflow: hidden\">\n",
       "<defs>\n",
       "<symbol id=\"arrow-right\" viewBox=\"0 0 330 330\">\n",
       "  <path id=\"XMLID_222_\" d=\"M250.606,154.389l-150-149.996c-5.857-5.858-15.355-5.858-21.213,0.001\n",
       "\tc-5.857,5.858-5.857,15.355,0.001,21.213l139.393,139.39L79.393,304.394c-5.857,5.858-5.857,15.355,0.001,21.213\n",
       "\tC82.322,328.536,86.161,330,90,330s7.678-1.464,10.607-4.394l149.999-150.004c2.814-2.813,4.394-6.628,4.394-10.606\n",
       "\tC255,161.018,253.42,157.202,250.606,154.389z\"/>\n",
       "</defs>\n",
       "</svg>\n",
       "<style>/* CSS stylesheet for displaying Paidiverpy objects in jupyterlab.\n",
       " *\n",
       " */\n",
       "\n",
       ":root {\n",
       "  --ppy-font-color: var(--jp-content-font-color, rgba(0, 0, 0, 1));\n",
       "  --ppy-font-color-brown: var(--jp-content-font-color-brown, brown);\n",
       "  --ppy-font-color-olive: var(--jp-content-font-color-olive, olive);\n",
       "  --ppy-font-color-teal: var(--jp-content-font-color-teal, teal);\n",
       "  --ppy-font-color-blue: var(--jp-content-font-color-blue, blue);\n",
       "  --ppy-font-color-gray: var(--jp-content-font-color-gray, gray);\n",
       "  --ppy-border-color: var(--jp-border-color2, #e0e0e0);\n",
       "  --ppy-disabled-color: var(--jp-layout-color3, #bdbdbd);\n",
       "  --ppy-background-color: var(--jp-layout-color, white);\n",
       "  --ppy-background-color2: var(--jp-layout-color3, rgba(212, 212, 212, 0.15));\n",
       "}\n",
       "\n",
       "\n",
       "html[theme=\"dark\"],\n",
       "html[data-theme=\"dark\"],\n",
       "body[data-theme=\"dark\"],\n",
       "body.vscode-dark {\n",
       "  --ppy-font-color: rgba(255, 255, 255, 1);\n",
       "  --ppy-font-color-brown: brown;\n",
       "  --ppy-font-color-olive: olive;\n",
       "  --ppy-font-color-teal: teal;\n",
       "  --ppy-font-color-blue: blue;\n",
       "  --ppy-font-color-gray: gray;\n",
       "  --ppy-border-color: #1f1f1f;\n",
       "  --ppy-disabled-color: #515151;\n",
       "  --ppy-background-color: #111111;\n",
       "}\n",
       "\n",
       ".ppy-background-color {\n",
       "  background-color: var(--ppy-background-color) !important;\n",
       "}\n",
       "\n",
       ".ppy-font-color-brown {\n",
       "  color: var(--ppy-font-color-brown) !important;\n",
       "}\n",
       "\n",
       ".ppy-font-color {\n",
       "  color: var(--ppy-font-color) !important;\n",
       "}\n",
       ".ppy-h2 {\n",
       "  font-size: 20px !important;\n",
       "}\n",
       ".ppy-h3 {\n",
       "  font-size: 13px !important;\n",
       "}\n",
       ".ppy-p {\n",
       "  font-size: 10px;\n",
       "}\n",
       ".ppy-p-error {\n",
       "  color: var(--ppy-font-color-brown);\n",
       "  font-weight: bold;\n",
       "  display: block;\n",
       "}\n",
       "\n",
       ".ppy-text-repr-fallback {\n",
       "  /* fallback to plain text repr when CSS is not injected (untrusted notebook) */\n",
       "  display: none;\n",
       "}\n",
       "\n",
       ".ppy-pipeline-wrap{\n",
       "  display: flex;\n",
       "  flex-wrap: wrap;\n",
       "  align-items: center;\n",
       "}\n",
       "\n",
       ".ppy-pipeline {\n",
       "  float:left;\n",
       "  cursor: pointer;\n",
       "  padding: 10px;\n",
       "  width: max-content;\n",
       "  height: 150px;\n",
       "  margin: 10px;\n",
       "  border: 1px solid #000;\n",
       "  text-align: center;\n",
       "  /* line-height: 80px; */\n",
       "}\n",
       "\n",
       ".ppy-pipeline-clear-fix {\n",
       "  clear: both;\n",
       "}\n",
       "\n",
       ".ppy-pipeline-parameters-all {\n",
       "  padding: 10px;\n",
       "  margin: 10px;\n",
       "}\n",
       "\n",
       ".ppy-images-step-header {\n",
       "  cursor: pointer;\n",
       "  font-weight: bold;\n",
       "  margin-top: 10px;\n",
       "  margin-bottom: 5px;\n",
       "}\n",
       ".ppy-images-metadata {\n",
       "  margin-left: 20px;\n",
       "  margin-bottom: 10px;\n",
       "}\n",
       ".ppy-images {\n",
       "  display: flex;\n",
       "  flex-wrap: wrap;\n",
       "  gap: 10px;\n",
       "  margin-left: 20px;\n",
       "}\n",
       "\n",
       ".ppy-show-more-button, .ppy-hide-button {\n",
       "  margin-left: 20px;\n",
       "  background-color: #007bff;\n",
       "  color: white;\n",
       "  border: none;\n",
       "  padding: 5px 10px;\n",
       "  cursor: pointer;\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-pipeline-arrow {\n",
       "  float:left;\n",
       "  width: 50px;\n",
       "  height: 80px;\n",
       "  margin: 10px;\n",
       "  text-align: center;\n",
       "  line-height: 80px;\n",
       "}\n",
       "\n",
       ".ppy-images-img{\n",
       "  display: block;\n",
       "  max-width: 200px;\n",
       "  margin: 10px;\n",
       "}\n",
       "\n",
       ".ppy-images-toggle-arrow {\n",
       "  cursor: pointer;\n",
       "  margin-left: 5px;\n",
       "}\n",
       "\n",
       ".ppy-images-image-p {\n",
       "  cursor: pointer;\n",
       "}\n",
       "\n",
       ".ppy-arrow-right {\n",
       "  display: inline-block;\n",
       "  vertical-align: middle;\n",
       "  width: 1em;\n",
       "  height: 1.5em !important;\n",
       "  stroke-width: 0;\n",
       "  stroke: currentColor;\n",
       "  fill: currentColor;\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-json-block {\n",
       "  font-family: monospace;\n",
       "  background: var(--ppy-background-color);\n",
       "  padding: 1em;\n",
       "  border-radius: 6px;\n",
       "}\n",
       ".ppy-json-key {\n",
       "  color: var(--ppy-font-color-brown);\n",
       "}\n",
       ".ppy-json-string {\n",
       "  color: var(--ppy-font-color-olive);\n",
       "}\n",
       ".ppy-json-number {\n",
       "  color: var(--ppy-font-color-teal);\n",
       "}\n",
       ".ppy-json-bool {\n",
       "  color: var(--ppy-font-color-blue);\n",
       "}\n",
       ".ppy-json-null {\n",
       "  color: var(--ppy-font-color-gray);\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-pn-title-0 {\n",
       "  font-size: 30px;\n",
       "}\n",
       "\n",
       ".ppy-pn-title-1 {\n",
       "  font-size: 20px;\n",
       "}\n",
       ".ppy-pn-title-2 {\n",
       "  font-size: 17px;\n",
       "}\n",
       ".ppy-pn-title-3 {\n",
       "  font-size: 14px;\n",
       "}\n",
       ".ppy-pn-title-4 {\n",
       "  font-size: 12px;\n",
       "}\n",
       "\n",
       ".ppy-pn-bold {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".ppy-pn-italic {\n",
       "  font-style: italic;\n",
       "}\n",
       ".ppy-pn-description {\n",
       "  font-size: 14px;\n",
       "  margin-bottom: 1px;\n",
       "}\n",
       "\n",
       ".bk-root {\n",
       "  width: 100% !important;\n",
       "  max-width: 100% !important;\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "\n",
       "input, .bk-input, .bk-input-group {\n",
       "  margin-top: 0px;\n",
       "  margin-bottom: 3px;\n",
       "  padding-top: 0px;\n",
       "  padding-bottom: 0px;\n",
       "}\n",
       "\n",
       ".ppy-pn-header-button .bk-btn {\n",
       "  background: none !important;\n",
       "  border: none !important;\n",
       "  font-weight: bold;\n",
       "  font-size: 16px;\n",
       "  text-align: left;\n",
       "  padding-left: 0;\n",
       "  cursor: pointer;\n",
       "  box-shadow: none !important;\n",
       "  color: black;\n",
       "}\n",
       "\n",
       ".ppy-pn-danger-modal{\n",
       "  border: 2px solid red;\n",
       "  border-radius: 10px;\n",
       "  box-shadow: 0px 0px 10px rgba(255, 0, 0, 0.5);\n",
       "}\n",
       "\n",
       ".ppy-pn-config-form{\n",
       "  padding: 10px;\n",
       "  margin-top: 10px;\n",
       "  margin-bottom: 10px;\n",
       "  background-color: var(--ppy-background-color2);\n",
       "  border-radius: 5px;\n",
       "  box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.1);\n",
       "}\n",
       "\n",
       "\n",
       ".ppy-pn-align-right {\n",
       "  display: flex;\n",
       "  justify-content: flex-end;\n",
       "}\n",
       "</style><script>function toggleMetadata(stepIndex, randomId) {\n",
       "  var metadata = document.getElementById(\"ppy-images-metadata-\" + randomId + \"-\" + stepIndex);\n",
       "  var arrow = document.getElementById(\"ppy-images-arrow-\" + randomId + \"-\" + stepIndex);\n",
       "  if (metadata.style.display === \"none\") {\n",
       "      metadata.style.display = \"block\";\n",
       "      arrow.innerHTML = \"▼\";\n",
       "      arrow.classList.remove(\"ppy-font-color\");\n",
       "      arrow.classList.add(\"ppy-font-color-brown\");\n",
       "  } else {\n",
       "      metadata.style.display = \"none\";\n",
       "      arrow.innerHTML = \"►\";\n",
       "      arrow.classList.remove(\"ppy-font-color-brown\");\n",
       "      arrow.classList.add(\"ppy-font-color\");\n",
       "  }\n",
       "}\n",
       "function toggleImage(imageId, randomId) {\n",
       "  var img = document.getElementById(imageId);\n",
       "  var arrow = document.getElementById(\"ppy-images-arrow-\" + randomId + \"-\" + imageId);\n",
       "  if (img.style.display === \"none\") {\n",
       "      img.style.display = \"block\";\n",
       "      arrow.innerHTML = \"▼\";\n",
       "      arrow.classList.remove(\"ppy-font-color\");\n",
       "      arrow.classList.add(\"ppy-font-color-brown\");\n",
       "  } else {\n",
       "      img.style.display = \"none\";\n",
       "      arrow.innerHTML = \"►\";\n",
       "      arrow.classList.remove(\"ppy-font-color-brown\");\n",
       "      arrow.classList.add(\"ppy-font-color\");\n",
       "  }\n",
       "}\n",
       "function showMore(stepIndex) {\n",
       "  var moreImages = document.getElementById(\"ppy-more-images-\" + stepIndex);\n",
       "  var showMoreButton = document.getElementById(\"ppy-show-more-button-\" + stepIndex);\n",
       "  var hideButton = document.getElementById(\"ppy-hide-button-\" + stepIndex);\n",
       "  moreImages.style.display = \"flex\";\n",
       "  showMoreButton.style.display = \"none\";\n",
       "  hideButton.style.display = \"inline-block\";\n",
       "}\n",
       "function hide(stepIndex) {\n",
       "  var moreImages = document.getElementById(\"ppy-more-images-\" + stepIndex);\n",
       "  var showMoreButton = document.getElementById(\"ppy-show-more-button-\" + stepIndex);\n",
       "  var hideButton = document.getElementById(\"ppy-hide-button-\" + stepIndex);\n",
       "  moreImages.style.display = \"none\";\n",
       "  showMoreButton.style.display = \"inline-block\";\n",
       "  hideButton.style.display = \"none\";\n",
       "}\n",
       "\n",
       "\n",
       "function showParameters(id, randomId) {\n",
       "  var currentTarget = document.getElementById('ppy-pipeline-' + randomId + '-' + id);\n",
       "  var square = document.getElementsByClassName('ppy-pipeline-' + randomId);\n",
       "  var allParams = document.getElementsByClassName('ppy-pipeline-parameters-' + randomId);\n",
       "  var selectedParams = document.getElementById('ppy-pipeline-parameters-' + randomId + '-' + id);\n",
       "  var idWasVisible = false;\n",
       "  if (selectedParams) {\n",
       "      var idWasVisible = selectedParams.style.display === 'block';\n",
       "  }\n",
       "  for (var i = 0; i < square.length; i++) {\n",
       "      square[i].classList.remove('ppy-font-color-brown');\n",
       "      square[i].classList.add('ppy-font-color');\n",
       "  }\n",
       "  for (var i = 0; i < allParams.length; i++) {\n",
       "      allParams[i].style.display = 'none';\n",
       "  }\n",
       "  // Show the selected parameter section\n",
       "  if (selectedParams) {\n",
       "      if (idWasVisible) {\n",
       "          selectedParams.style.display = 'none';\n",
       "          currentTarget.classList.remove('ppy-font-color-brown');\n",
       "          currentTarget.classList.add('ppy-font-color');\n",
       "      } else {\n",
       "          selectedParams.style.display = 'block';\n",
       "          currentTarget.classList.remove('ppy-font-color');\n",
       "          currentTarget.classList.add('ppy-font-color-brown');\n",
       "      }\n",
       "  }\n",
       "}\n",
       "\n",
       "hljs.highlightAll();\n",
       "</script><link rel=\"stylesheet\" href=\"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css\"><script src=\"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js\"></script><script>hljs.highlightAll();</script><pre class='ppy-text-repr-fallback'>&lt;paidiverpy.pipeline.pipeline.Pipeline object at 0x7d251c6cffd0&gt;</pre><div>\n",
       "    <div class=\"ppy-pipeline-wrap\">\n",
       "        \n",
       "    <div id=\"ppy-pipeline-pbe-general\" title=\"Click to see more information\"\n",
       "        class=\"ppy-pipeline ppy-pipeline-pbe\"\n",
       "        onclick=\"showParameters('general', 'pbe')\">\n",
       "        <h2 class=\"ppy-h2\">Raw</h2>\n",
       "        <h3 class=\"ppy-h3\">Type: Open</h2>\n",
       "    </div>\n",
       "    <div class=\"ppy-pipeline-arrow\"><svg class='icon ppy-arrow-right'><use xlink:href='#arrow-right'></use></svg></div>\n",
       "            <div id=\"ppy-pipeline-pbe-step-0\" title=\"Click to see more information\"\n",
       "                class=\"ppy-pipeline ppy-pipeline-pbe\" onclick=\"showParameters('step-0', 'pbe')\">\n",
       "                <h2 class=\"ppy-h2\">Datetime</h2>\n",
       "                <h3 class=\"ppy-h3\">Type: Sampling</h2>\n",
       "            </div>\n",
       "        <div class='ppy-pipeline-arrow'><svg class='icon ppy-arrow-right'><use xlink:href='#arrow-right'></use></svg></div>\n",
       "            <div id=\"ppy-pipeline-pbe-step-1\" title=\"Click to see more information\"\n",
       "                class=\"ppy-pipeline ppy-pipeline-pbe\" onclick=\"showParameters('step-1', 'pbe')\">\n",
       "                <h2 class=\"ppy-h2\">Overlapping</h2>\n",
       "                <h3 class=\"ppy-h3\">Type: Sampling</h2>\n",
       "            </div>\n",
       "        <div class='ppy-pipeline-arrow'><svg class='icon ppy-arrow-right'><use xlink:href='#arrow-right'></use></svg></div>\n",
       "            <div id=\"ppy-pipeline-pbe-step-2\" title=\"Click to see more information\"\n",
       "                class=\"ppy-pipeline ppy-pipeline-pbe\" onclick=\"showParameters('step-2', 'pbe')\">\n",
       "                <h2 class=\"ppy-h2\">Colour_alteration</h2>\n",
       "                <h3 class=\"ppy-h3\">Type: Colour</h2>\n",
       "            </div>\n",
       "        <div class='ppy-pipeline-arrow'><svg class='icon ppy-arrow-right'><use xlink:href='#arrow-right'></use></svg></div>\n",
       "            <div id=\"ppy-pipeline-pbe-step-3\" title=\"Click to see more information\"\n",
       "                class=\"ppy-pipeline ppy-pipeline-pbe\" onclick=\"showParameters('step-3', 'pbe')\">\n",
       "                <h2 class=\"ppy-h2\">Gaussian_blur</h2>\n",
       "                <h3 class=\"ppy-h3\">Type: Colour</h2>\n",
       "            </div>\n",
       "        <div class='ppy-pipeline-arrow'><svg class='icon ppy-arrow-right'><use xlink:href='#arrow-right'></use></svg></div><div class=\"ppy-pipeline-clear-fix\"></div>\n",
       "            <div id=\"ppy-pipeline-pbe-step-4\" title=\"Click to see more information\"\n",
       "                class=\"ppy-pipeline ppy-pipeline-pbe\" onclick=\"showParameters('step-4', 'pbe')\">\n",
       "                <h2 class=\"ppy-h2\">Sharpen</h2>\n",
       "                <h3 class=\"ppy-h3\">Type: Colour</h2>\n",
       "            </div>\n",
       "        <div class='ppy-pipeline-arrow'><svg class='icon ppy-arrow-right'><use xlink:href='#arrow-right'></use></svg></div>\n",
       "            <div id=\"ppy-pipeline-pbe-step-5\" title=\"Click to see more information\"\n",
       "                class=\"ppy-pipeline ppy-pipeline-pbe\" onclick=\"showParameters('step-5', 'pbe')\">\n",
       "                <h2 class=\"ppy-h2\">Contrast</h2>\n",
       "                <h3 class=\"ppy-h3\">Type: Colour</h2>\n",
       "            </div>\n",
       "        \n",
       "    </div>\n",
       "    <div id=\"ppy-pipeline-parameters\" class=\"ppy-pipeline-parameters-all\">\n",
       "            <div id=\"ppy-pipeline-parameters-pbe-step-0\"\n",
       "                class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "                style=\"display: none;\">\n",
       "                <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "                <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"datetime\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "<span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">params</span>:\n",
       "  <span class=\"ppy-json-key\">min</span>: <span class=\"ppy-json-string\">\"2018-06-11 04:14:00\"</span>\n",
       "  <span class=\"ppy-json-key\">max</span>: <span class=\"ppy-json-string\">\"2018-06-11 04:20:00\"</span>\n",
       "  <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"datetime\"</span></pre>\n",
       "            </div>\n",
       "        \n",
       "            <div id=\"ppy-pipeline-parameters-pbe-step-1\"\n",
       "                class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "                style=\"display: none;\">\n",
       "                <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "                <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"overlapping\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "<span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">params</span>:\n",
       "  <span class=\"ppy-json-key\">omega</span>: <span class=\"ppy-json-number\">57.0</span>\n",
       "  <span class=\"ppy-json-key\">theta</span>: <span class=\"ppy-json-number\">40.0</span>\n",
       "  <span class=\"ppy-json-key\">threshold</span>: <span class=\"ppy-json-number\">0.5</span>\n",
       "  <span class=\"ppy-json-key\">camera_distance</span>: <span class=\"ppy-json-number\">1.12</span>\n",
       "  <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"overlapping\"</span></pre>\n",
       "            </div>\n",
       "        \n",
       "            <div id=\"ppy-pipeline-parameters-pbe-step-2\"\n",
       "                class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "                style=\"display: none;\">\n",
       "                <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "                <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"colour_alteration\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "<span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">params</span>:\n",
       "  <span class=\"ppy-json-key\">method</span>: <span class=\"ppy-json-string\">\"white_balance\"</span>\n",
       "  <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"colour_alteration\"</span></pre>\n",
       "            </div>\n",
       "        \n",
       "            <div id=\"ppy-pipeline-parameters-pbe-step-3\"\n",
       "                class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "                style=\"display: none;\">\n",
       "                <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "                <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"gaussian_blur\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "<span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">params</span>:\n",
       "  <span class=\"ppy-json-key\">sigma</span>: <span class=\"ppy-json-number\">1.0</span>\n",
       "  <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"gaussian_blur\"</span></pre>\n",
       "            </div>\n",
       "        \n",
       "            <div id=\"ppy-pipeline-parameters-pbe-step-4\"\n",
       "                class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "                style=\"display: none;\">\n",
       "                <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "                <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"sharpen\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "<span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">params</span>:\n",
       "  <span class=\"ppy-json-key\">alpha</span>: <span class=\"ppy-json-number\">1.5</span>\n",
       "  <span class=\"ppy-json-key\">beta</span>: <span class=\"ppy-json-number\">-0.5</span>\n",
       "  <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"sharpen\"</span></pre>\n",
       "            </div>\n",
       "        \n",
       "            <div id=\"ppy-pipeline-parameters-pbe-step-5\"\n",
       "                class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "                style=\"display: none;\">\n",
       "                <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "                <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"contrast\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"colour\"</span>\n",
       "<span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">params</span>:\n",
       "  <span class=\"ppy-json-key\">method</span>: <span class=\"ppy-json-string\">\"clahe\"</span>\n",
       "  <span class=\"ppy-json-key\">clip_limit</span>: <span class=\"ppy-json-number\">0.01</span>\n",
       "  <span class=\"ppy-json-key\">gamma_value</span>: <span class=\"ppy-json-number\">0.5</span>\n",
       "  <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "<span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"contrast\"</span></pre>\n",
       "            </div>\n",
       "        \n",
       "        <div id=\"ppy-pipeline-parameters-pbe-general\" class=\"ppy-pipeline-parameters ppy-pipeline-parameters-pbe\"\n",
       "            style=\"display: none;\">\n",
       "            <h4 class=\"ppy-h4\">Parameters:</h4>\n",
       "            <pre class=\"ppy-json-block\"><span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"raw\"</span>\n",
       "<span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"open\"</span>\n",
       "<span class=\"ppy-json-key\">sample_data</span>: <span class=\"ppy-json-string\">\"benthic_csv\"</span>\n",
       "<span class=\"ppy-json-key\">input_path</span>: <span class=\"ppy-json-string\">\"/home/tobfer/.paidiverpy_cache/benthic_csv/images\"</span>\n",
       "<span class=\"ppy-json-key\">output_path</span>: <span class=\"ppy-json-string\">\"output\"</span>\n",
       "<span class=\"ppy-json-key\">metadata_path</span>: <span class=\"ppy-json-string\">\"/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/metadata_benthic_csv.csv\"</span>\n",
       "<span class=\"ppy-json-key\">metadata_type</span>: <span class=\"ppy-json-string\">\"CSV_FILE\"</span>\n",
       "<span class=\"ppy-json-key\">image_open_args</span>: <span class=\"ppy-json-string\">\"PNG\"</span>\n",
       "<span class=\"ppy-json-key\">append_data_to_metadata</span>: <span class=\"ppy-json-string\">\"/home/tobfer/.paidiverpy_cache/benthic_csv/metadata/appended_metadata_benthic_csv.csv\"</span>\n",
       "<span class=\"ppy-json-key\">n_jobs</span>: <span class=\"ppy-json-number\">-1</span>\n",
       "<span class=\"ppy-json-key\">track_changes</span>: <span class=\"ppy-json-number\">True</span>\n",
       "<span class=\"ppy-json-key\">sampling</span>:\n",
       "  - <span class=\"ppy-json-key\">name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "    <span class=\"ppy-json-key\">step_name</span>: <span class=\"ppy-json-string\">\"sampling\"</span>\n",
       "    <span class=\"ppy-json-key\">test</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">params</span>:\n",
       "      <span class=\"ppy-json-key\">value</span>: <span class=\"ppy-json-number\">20</span>\n",
       "      <span class=\"ppy-json-key\">raise_error</span>: <span class=\"ppy-json-number\">False</span>\n",
       "    <span class=\"ppy-json-key\">mode</span>: <span class=\"ppy-json-string\">\"fixed\"</span></pre>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div></div>"
      ],
      "text/plain": [
       "<paidiverpy.pipeline.pipeline.Pipeline at 0x7d251c6cffd0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# See the pipeline steps. Click in a step to see more information about it\n",
    "pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e61c25b4-79fd-4f28-94bb-e68aab8d0e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "# See the pipeline steps in json format\n",
    "# pipeline.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0131f808-2a6a-40e0-beb7-41726f5c638a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:26 | Processing images using 16 cores\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Processing images using 16 cores\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:26 | Running step 0: raw - OpenLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 0: raw - OpenLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed | 103.79 ms\n",
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:27 | Step 0 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 0 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:27 | Running step 1: datetime - SamplingLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 1: datetime - SamplingLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:27 | Number of images before the sampling step: 20. Total number of images after: 14\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Number of images before the sampling step: 20. Total number of images after: 14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:27 | Step 1 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 1 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:27 | Running step 2: overlapping - SamplingLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 2: overlapping - SamplingLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:27 | Number of images before the sampling step: 14. Total number of images after: 9\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Number of images before the sampling step: 14. Total number of images after: 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Step 2 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 2 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Running step 3: colour_alteration - ColourLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 3: colour_alteration - ColourLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Step 3 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 3 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Running step 4: gaussian_blur - ColourLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 4: gaussian_blur - ColourLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Step 4 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 4 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Running step 5: sharpen - ColourLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 5: sharpen - ColourLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Step 5 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 5 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:28 | Running step 6: contrast - ColourLayer\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Running step 6: contrast - ColourLayer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m☁ paidiverpy ☁  |       INFO | 2025-09-11 16:48:29 | Step 6 completed\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:paidiverpy:Step 6 completed\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[########################################] | 100% Completed | 1.06 sms\n"
     ]
    }
   ],
   "source": [
    "# Run the pipeline\n",
    "pipeline.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a09a52-bbe4-4f4c-b645-607ad8d58878",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
