{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Molecular dynamics application example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-requirements"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-08T11:04:16.725553Z",
     "start_time": "2020-06-08T11:04:16.151957Z"
    }
   },
   "outputs": [],
   "source": [
    "# Primary imports\n",
    "import importlib     # Only needed for module editing\n",
    "import json\n",
    "import pandas as pd  # Optional dependency\n",
    "from pathlib import Path\n",
    "import pprint\n",
    "import sys\n",
    "import time\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")  # Surpress or enable warnings\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from sklearn import datasets  # For sklearn test data set creation\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# CNN clustering module\n",
    "import cnnclustering.cnn as cnn    # CNN clustering\n",
    "import cnnclustering.cmsm as cmsm  # Core-set MSM estimation\n",
    "import pydpc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook was created using Python 3.8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-08T09:14:22.691008Z",
     "start_time": "2020-06-08T09:14:22.685971Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.8.5 | packaged by conda-forge | (default, Aug 21 2020, 18:21:27) \n",
      "[GCC 7.5.0]\n"
     ]
    }
   ],
   "source": [
    "# Version information\n",
    "print(sys.version)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook configuration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use `matplotlib` to create plots. A `\"matplotlibrc\"` file can be used to customise the appearance of the plots."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-08T11:04:19.413868Z",
     "start_time": "2020-06-08T11:04:19.401710Z"
    }
   },
   "outputs": [],
   "source": [
    "# Matplotlib configuration\n",
    "mpl.rc_file(\n",
    "    \"../matplotlibrc\",\n",
    "    use_default_template=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-08T11:04:19.629939Z",
     "start_time": "2020-06-08T11:04:19.619848Z"
    }
   },
   "outputs": [],
   "source": [
    "# Axis property defaults for the plots\n",
    "ax_props = {\n",
    "    \"xlabel\": None,\n",
    "    \"ylabel\": None,\n",
    "    \"xlim\": (-2.5, 2.5),\n",
    "    \"ylim\": (-2.5, 2.5),\n",
    "    \"xticks\": (),\n",
    "    \"yticks\": (),\n",
    "    \"aspect\": \"equal\"\n",
    "}\n",
    "\n",
    "# Line plot property defaults\n",
    "line_props = {\n",
    "    \"linewidth\": 0,\n",
    "    \"marker\": '.',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Package configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-08T09:14:44.838616Z",
     "start_time": "2020-06-08T09:14:44.836274Z"
    }
   },
   "outputs": [],
   "source": [
    "# Configuration file found?\n",
    "cnn.settings.cfgfile  # If None, no file is provided"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-08T09:14:45.318558Z",
     "start_time": "2020-06-08T09:14:45.305150Z"
    },
    "run_control": {
     "marked": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'default_cnn_cutoff': '1',\n",
       " 'default_cnn_offset': '0',\n",
       " 'default_radius_cutoff': '1',\n",
       " 'default_member_cutoff': '2',\n",
       " 'default_fit_policy': 'conservative',\n",
       " 'default_predict_policy': 'conservative',\n",
       " 'float_precision': 'sp',\n",
       " 'int_precision': 'sp'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display default settings\n",
    "cnn.settings.defaults"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_evaluate(clusterobject, axis_labels=False, plot=\"dots\"):\n",
    "    fig, Ax = plt.subplots(\n",
    "        1, 3,\n",
    "        figsize=(mpl.rcParams['figure.figsize'][0],\n",
    "                 mpl.rcParams['figure.figsize'][1]*0.5)\n",
    "        )\n",
    "\n",
    "    for dim in range(3):\n",
    "        dim_ = (dim * 2, dim * 2 + 1)\n",
    "        \n",
    "        ax_props_ = {k: v for k, v in ax_props.items()}\n",
    "        if axis_labels:\n",
    "            ax_props_.update({\"xlabel\": dim_[0] + 1, \"ylabel\": dim_[1] + 1})\n",
    "\n",
    "        _ = clusterobject.evaluate(\n",
    "            ax=Ax[dim], plot=plot,\n",
    "            ax_props=ax_props_,\n",
    "            dim=dim_\n",
    "            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MD showcase - Langerin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's read in some \"real world\" data for this example. We will work with a 6D projection of a classical MD trajectory of the C-type lectin receptor langerin that was generated by the dimension reduction procedure TICA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:15:38.246335Z",
     "start_time": "2020-05-15T10:15:38.234503Z"
    }
   },
   "outputs": [],
   "source": [
    "langerin = cnn.CNN(points=np.load(\"md_example/md_showcase_langerin.npy\", allow_pickle=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After creating a `CNN` instance, we can print out basic information about the data. The projection comes in 116 parts of individual independent simulations. The number of data points in these parts is written out for the first 4 parts. In total we have about 2.6 million data points in this set representing 26 microseconds of simulation time at a sampling timestep of 10 picoseconds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "CNN cluster object\n",
      "--------------------------------------------------------------------------------\n",
      "Alias :                         root\n",
      "Hierachy level :                0\n",
      "\n",
      "Data point shape :              Parts      - 116, [5571  4148  20851  99928 ...]\n",
      "                                Points     - 2641593\n",
      "                                Dimensions - 6\n",
      "\n",
      "Distance matrix calculated :    None\n",
      "Neighbourhoods calculated :     None\n",
      "Density graph calculated :      None\n",
      "\n",
      "Clustered :                     False\n",
      "Children :                      False\n",
      "================================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(langerin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dealing with six data dimensions we can still visualise the data quite well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 19912 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin, axis_labels=True, plot=\"contourf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clustering of this fairly large amount of data points directly is in principle possible, but it will be slow. Pre-calculating pairwise distances would occupy terabytes of disk space which we can not afford and so we have resort to brute-force on-the-fly distance calculation. To allow quick and handy data exploration and cluster result screening we want to work on a reduced data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced = langerin.cut(points=(None, None, 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we distance pre-calculation is feasible and clustering will be much faster. When reducing a data set it is most important that the new set remains representative of the original one. Usually using a regular stride on the data points will be appropriate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24548 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced, axis_labels=True, plot=\"contourf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "CNN cluster object\n",
      "--------------------------------------------------------------------------------\n",
      "Alias :                         root\n",
      "Hierachy level :                0\n",
      "\n",
      "Data point shape :              Parts      - 116, [56  42  209  1000 ...]\n",
      "                                Points     - 26528\n",
      "                                Dimensions - 6\n",
      "\n",
      "Distance matrix calculated :    None\n",
      "Neighbourhoods calculated :     None\n",
      "Density graph calculated :      None\n",
      "\n",
      "Clustered :                     False\n",
      "Children :                      False\n",
      "================================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A quick look on the distribution of distances in the set gives us a first feeling for what might be a suitable value for the neighbour search radius *r*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:16:32.484528Z",
     "start_time": "2020-05-15T10:16:25.493386Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Mapping: 100%|██████████| 6.00/6.00 [00:09<00:00, 1.52s/Chunks]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<Figure size 750x450 with 1 Axes>,\n",
       " <AxesSubplot:xlabel='d / au'>,\n",
       " [<matplotlib.lines.Line2D at 0x7f147d2eddf0>],\n",
       " None)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 21684 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin_reduced.calc_dist(mmap=True, chunksize=5000)  # Pre-calculate point distances and temporary store on disk\n",
    "langerin_reduced.dist_hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering root data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can expect a split of the data into clusters for values of *r* of roughly 2 or lower. Let's attempt a first clustering step with a relatively low density criterion (large *r* cutoff, low number of common neighbours *c*):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 23.0190 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "26528     2.000     5         2         None      3         0.983     0.000     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "langerin_reduced.fit(2, 5, policy=\"progressive\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:19:25.993477Z",
     "start_time": "2020-05-15T10:19:25.652620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 30964 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that no data point was excluded as sparse outlier and we split the data into three clusters. Especially the first cluster can obviously be splitted further. Let' recluster it applying a higher density criterion. For this we first need to freeze this cluster result and isolate the obtained clusters into individual child cluster objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.isolate()\n",
    "current = c1 = langerin_reduced.children[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, the isolation assigns an alias to each child cluster that reveals its origin. In this case \"root - 1\", translates to the first child cluster of the root data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:22:21.659782Z",
     "start_time": "2020-05-15T10:22:06.533636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 1.4944 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "26079     1.000     5         2         None      3         0.727     0.000     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()              # We pre-compute neighbourhoods here\n",
    "r = 1\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:19:25.993477Z",
     "start_time": "2020-05-15T10:19:25.652620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 33840 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The re-clustered data points are split into another three clusters. This time, we see an oportunity to again re-cluster the first two obtained clusters, slightly increasing the density criterion further."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "current.isolate()\n",
    "current = c1_1 = current.children[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1 - 1)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this stage we choose a `member_cutoff` of 10 for the fit, ensuring that we do not yiels small, meaningless clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:22:21.659782Z",
     "start_time": "2020-05-15T10:22:06.533636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.4998 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "18947     0.450     15        10        None      3         0.946     0.027     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.45\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 15, member_cutoff=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:19:25.993477Z",
     "start_time": "2020-05-15T10:19:25.652620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 30900 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "current = c1_2 = c1.children[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1 - 2)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.0964 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "7077      0.500     10        2         None      3         0.613     0.010     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Recluster second cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.5\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23044 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And on it goes ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "current = c1_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "current.isolate()\n",
    "current = c1_1_1 = current.children[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1 - 1 - 1)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:22:21.659782Z",
     "start_time": "2020-05-15T10:22:06.533636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.3754 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "17927     0.400     15        2         None      2         0.867     0.015     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.4\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:19:25.993477Z",
     "start_time": "2020-05-15T10:19:25.652620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 21768 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "current.isolate()\n",
    "current = c1_1_1_1 = current.children[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1 - 1 - 1 - 1)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:22:21.659782Z",
     "start_time": "2020-05-15T10:22:06.533636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.1906 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "15548     0.280     15        10        None      2         0.676     0.142     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.28\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 15, member_cutoff=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:19:25.993477Z",
     "start_time": "2020-05-15T10:19:25.652620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24848 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "current.isolate()\n",
    "current = c1_1_1_1_1 = current.children[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1 - 1 - 1 - 1 - 1)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:22:21.659782Z",
     "start_time": "2020-05-15T10:22:06.533636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.1533 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "10506     0.220     15        10        None      2         0.650     0.309     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.22\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 15, member_cutoff=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:19:25.993477Z",
     "start_time": "2020-05-15T10:19:25.652620Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 20432 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge hierarchy levels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to leave it with that for the moment. We can visualise our cluster hierarchy to get an overview."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 65604 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = langerin_reduced.pie()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we can finally put everything together and incorporate the child clusters into the root data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.reel(deep=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After this call, cluster labeling may not be contiguous and sorted by size, which we can fix easily."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.labels.sort_by_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 67780 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label    Size\n",
      "=============\n",
      "(0, 6326)\n",
      "(1, 6831)\n",
      "(2, 4335)\n",
      "(3, 2834)\n",
      "(4, 2619)\n",
      "(5, 2104)\n",
      "(6, 451)\n",
      "(7, 425)\n",
      "(8, 231)\n",
      "(9, 217)\n",
      "(10, 55)\n",
      "(11, 52)\n",
      "(12, 48)\n"
     ]
    }
   ],
   "source": [
    "print(\"Label    Size\")\n",
    "print(\"=============\")\n",
    "print(*sorted({k: len(v) for k, v in langerin_reduced.labels.clusterdict.items()}.items()), sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For later re-use, we save the clustering result in form of the cluster label assignments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"md_example/cluster_labels.npy\", np.asarray(langerin_reduced.labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MSM estimation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assuming our data was sampled in a time-correlated manner as it is the case for MD simulation data, we can use this clustering result as a basis for the estimation of a core-set Markov-state model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = cmsm.CMSM(langerin_reduced.get_dtraj(), unit=\"ns\", step=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 1 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 26380 steps over 12 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 2 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 26380 steps over 12 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 4 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 26380 steps over 12 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 8 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 26380 steps over 12 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 15 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Trajectories [0, 1]\n",
      "are shorter then step threshold (lag*minlenfactor = 75)\n",
      "and will not be used to compute the MSM.\n",
      "\n",
      "Using 114 trajectories with 26284 steps over 12 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 30 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Trajectories [0, 1, 4, 63]\n",
      "are shorter then step threshold (lag*minlenfactor = 150)\n",
      "and will not be used to compute the MSM.\n",
      "\n",
      "Using 112 trajectories with 25999 steps over 12 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Estimate csMSM for different lag times (given in steps)\n",
    "lags = [1, 2, 4, 8, 15, 30]\n",
    "for i in lags:\n",
    "    M.cmsm(lag=i, minlenfactor=5)\n",
    "    M.get_its()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 49268 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the time scales\n",
    "fig, ax, *_ = M.plot_its()\n",
    "fig.tight_layout(pad=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster alternatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32548 characters]\n",
      "text/plain": [
       "<Figure size 750x750 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pydpc_clustering = pydpc.Cluster(langerin_reduced.data.points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pydpc_clustering.autoplot = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pydpc_clustering.assign(0, 1.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  690,  5274,  5470, 11430, 14034, 16202, 17564], dtype=int32)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pydpc_clustering.clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.labels = (pydpc_clustering.membership + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41768 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.labels[pydpc_clustering.halo_idx] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58248 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = cmsm.CMSM(langerin_reduced.get_dtraj(), unit=\"ns\", step=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T09:31:02.428514Z",
     "start_time": "2019-09-12T09:31:02.357788Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 1 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 2 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 4 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 8 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 15 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Trajectories [0, 1, 73]\n",
      "are shorter then step threshold (lag * minlenfactor = 75)\n",
      "and will not be used to compute the MSM.\n",
      "\n",
      "Using 113 trajectories with 25732 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 30 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Trajectories [0, 1, 4, 63, 73]\n",
      "are shorter then step threshold (lag * minlenfactor = 150)\n",
      "and will not be used to compute the MSM.\n",
      "\n",
      "Using 111 trajectories with 25447 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Estimate csMSM for different lag times (given in steps)\n",
    "lags = [1, 2, 4, 8, 15, 30]\n",
    "for i in lags:\n",
    "    M.cmsm(lag=i, minlenfactor=5)\n",
    "    M.get_its()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T09:31:20.706043Z",
     "start_time": "2019-09-12T09:31:19.851168Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45388 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the time scales\n",
    "fig, ax, *_ = M.plot_its()\n",
    "fig.tight_layout(pad=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56724 characters]\n",
      "text/plain": [
       "<Figure size 750x630 with 7 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize = mpl.rcParams[\"figure.figsize\"]\n",
    "mpl.rcParams[\"figure.figsize\"] = figsize[0], figsize[1] * 0.2\n",
    "M.plot_eigenvectors()\n",
    "mpl.rcParams[\"figure.figsize\"] = figsize"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "164.988px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "toc-autonumbering": false,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false,
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
