{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Molecular dynamics application example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:20:25.111486Z",
     "start_time": "2021-03-23T17:20:25.105283Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from sklearn import datasets\n",
    "from sklearn.metrics import pairwise_distances\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from cnnclustering import cluster\n",
    "from cnnclustering import plot\n",
    "import csmsm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook was created using Python 3.8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T15:35:51.377616Z",
     "start_time": "2021-03-23T15:35:51.373548Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.8.8 (default, Mar 11 2021, 08:58:19) \n",
      "[GCC 8.3.0]\n"
     ]
    }
   ],
   "source": [
    "# Version information\n",
    "print(sys.version)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T15:36:02.356864Z",
     "start_time": "2021-03-23T15:36:02.350256Z"
    }
   },
   "outputs": [],
   "source": [
    "# Matplotlib configuration\n",
    "mpl.rc_file(\n",
    "    \"../../matplotlibrc\",\n",
    "    use_default_template=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T15:36:02.548448Z",
     "start_time": "2021-03-23T15:36:02.542857Z"
    }
   },
   "outputs": [],
   "source": [
    "# Axis property defaults for the plots\n",
    "ax_props = {\n",
    "    \"xlabel\": None,\n",
    "    \"ylabel\": None,\n",
    "    \"xlim\": (-2.5, 2.5),\n",
    "    \"ylim\": (-2.5, 2.5),\n",
    "    \"xticks\": (),\n",
    "    \"yticks\": (),\n",
    "    \"aspect\": \"equal\"\n",
    "}\n",
    "\n",
    "# Line plot property defaults\n",
    "line_props = {\n",
    "    \"linewidth\": 0,\n",
    "    \"marker\": '.',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:11:45.747232Z",
     "start_time": "2021-03-23T17:11:45.737719Z"
    }
   },
   "outputs": [],
   "source": [
    "def draw_evaluate(clustering, axis_labels=False, plot_style=\"dots\"):\n",
    "    fig, Ax = plt.subplots(\n",
    "        1, 3,\n",
    "        figsize=(mpl.rcParams['figure.figsize'][0],\n",
    "                 mpl.rcParams['figure.figsize'][1] * 0.5)\n",
    "        )\n",
    "\n",
    "    for dim in range(3):\n",
    "        dim_ = (dim * 2, dim * 2 + 1)\n",
    "        \n",
    "        ax_props_ = {k: v for k, v in ax_props.items()}\n",
    "        if axis_labels:\n",
    "            ax_props_.update(\n",
    "                {\"xlabel\": dim_[0] + 1, \"ylabel\": dim_[1] + 1}\n",
    "            )\n",
    "\n",
    "        _ = clustering.evaluate(\n",
    "            ax=Ax[dim],\n",
    "            plot_style=plot_style,\n",
    "            ax_props=ax_props_,\n",
    "            dim=dim_\n",
    "            )\n",
    "        \n",
    "        Ax[dim].yaxis.get_label().set_rotation(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The C-type lection receptor langerin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's read in some \"real world\" data for this example. We will work with a 6D projection from a classical MD trajectory of the C-type lectin receptor langerin that was generated by the dimension reduction procedure TICA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:16:18.276155Z",
     "start_time": "2021-03-23T17:16:18.205461Z"
    }
   },
   "outputs": [],
   "source": [
    "langerin = cluster.prepare_clustering(\n",
    "    np.load(\"md_example/langerin_projection.npy\", allow_pickle=True)\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After creating a `Clustering` instance, we can print out basic information about the data. The projection comes in 116 parts of individual independent simulations. In total we have about 26,000 data points in this set representing 26 microseconds of simulation time at a sampling timestep of 1 nanoseconds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:16:59.670654Z",
     "start_time": "2021-03-23T17:16:59.665768Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hierarchy_level: 0\n",
      "input_data_kind: points\n",
      "points: 26528\n",
      "children: None\n"
     ]
    }
   ],
   "source": [
    "print(langerin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dealing with six data dimensions we can still visualise the data quite well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:17:01.182402Z",
     "start_time": "2021-03-23T17:17:00.984780Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24280 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin, axis_labels=True, plot_style=\"contourf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A quick look on the distribution of distances in the set gives us a first feeling for what might be a suitable value for the neighbour search radius *r*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:21:29.073866Z",
     "start_time": "2021-03-23T17:21:10.033839Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23456 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "distances = pairwise_distances(langerin.input_data)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = plot.plot_histogram(ax, distances.flatten(), maxima=True, maxima_props={\"order\": 5})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can expect a split of the data into clusters for values of *r* of roughly 2 or lower. The maximum in the distribution closest to 0 defines a reasonable value range for $r$. We can also get a feeling for what values *c* could take by checking the number of neighbours each point has for a given search radius."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-23T17:31:09.259441Z",
     "start_time": "2021-03-23T17:31:03.698850Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 70356 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "range_ = (-0.5, 1000.5)\n",
    "bins_ = len(range(int(range_[0] + 0.5), int(range_[1] - 0.5))) + 1\n",
    "fig, ax = plt.subplots(2, 1)\n",
    "radii = [0.2, 0.5, 0.75, 1]\n",
    "for radius in radii:\n",
    "    n_neighbours = [x[(x < radius ) & (x > 0)].shape[0] for x in distances]\n",
    "    H, e = np.histogram(\n",
    "        n_neighbours,\n",
    "        bins = bins_,\n",
    "        range=range_\n",
    "    )\n",
    "    e = (e[:-1] + e[1:]) / 2\n",
    "    ax[0].plot(e, H)\n",
    "    ax[1].plot(e, H)\n",
    "\n",
    "ax[0].set(**{\n",
    "    \"title\": \"Number of neighbours with respect to a radius\",\n",
    "    \"xlim\": (range_[0] + 0.5, (range_[1] - 0.5) / 100),\n",
    "    \"ylabel\": \"number of points                    \"\n",
    "    })\n",
    "ax[1].set(**{\n",
    "    \"xlim\": (1, range_[1] - 0.5),\n",
    "    \"xscale\": \"log\",\n",
    "    \"yscale\": \"log\",\n",
    "    \"xlabel\": \"number of neighbours\"\n",
    "    })\n",
    "\n",
    "ax[0].legend(radii, fancybox=False, framealpha=1, edgecolor=\"k\")\n",
    "\n",
    "fig.tight_layout(pad=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the radius value regime of 1 and lower the number of neighbours per point quickly approaches the order of number of points in the whole data set. We can look at these plots as an upper bound for *c* if *r* gets lower. Values of *c* larger than 100 wont make sense for radii below 0.5. We can also see that for a radius of 0.2, a lot of the data points has no neighbour at all. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering root data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's attempt a first clustering step with a relatively low density criterion (large *r* cutoff, low number of common neighbours *c*). When we cluster, we can judge the outcome by how many clusters we obtain and how many points end up as being classified as noise. For the first clustering steps, it can be a good idea to start with a low density cutoff and then increase it gradually. As we choose higher density criteria the number of resulting clusters will rise and so will the noise level. We need to find the point where we yield a large number of (reasonably sized) clusters without loosing relevant information in the noise. Keep in mind that the ideal number of clusters and noise ratio are not definable and highly dependend on the nature of the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-03-23T17:39:17.925Z"
    }
   },
   "outputs": [],
   "source": [
    "c = 20\n",
    "\n",
    "for r in [2, 1, 0.8]:\n",
    "    langerin.fit(r, c) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 50.9977 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "26528     2.000     5         2         None      3         0.983     0.000     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 30936 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin.fit(2, 5)\n",
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Low number of clusters and no noise points. We can push this a bit further."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 17.9091 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "26528     1.000     5         2         None      5         0.714     0.001     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 41376 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin_reduced.fit(1, 5, policy=\"progressive\")\n",
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two more clusters and still little noise. There is room for more. At this point we choose a `member_cutoff` of 10 for the fit, ensuring that we do not yield very small, meaningless clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 6.1926 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "26528     0.500     5         10        None      7         0.710     0.011     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 44572 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin_reduced.fit(0.5, 5, policy=\"progressive\", member_cutoff=10)\n",
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This looks good so far. There is no point in going any further at this stage. If we reduce *r* even more, we begin to loose clusters again into noise:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 2.9209 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "26528     0.400     5         10        None      9         0.681     0.029     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 52904 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin_reduced.fit(0.4, 5, policy=\"progressive\", member_cutoff=10)\n",
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We saw that parts of the data could be splitted further, but that increasing the density criterion leads to possible loss of relevant information in other data areas. Especially the first cluster can obviously be splitted further. Let' recluster it applying a higher density criterion. For this we first need to freeze this cluster result and isolate the obtained clusters into individual child cluster objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label\tr\tc\n",
      "--------------------\n",
      "1\t0.5\t5\n",
      "2\t0.5\t5\n",
      "3\t0.5\t5\n",
      "4\t0.5\t5\n",
      "5\t0.5\t5\n",
      "6\t0.5\t5\n",
      "7\t0.5\t5\n"
     ]
    }
   ],
   "source": [
    "# Lets make sure we work on the correctly clustered object\n",
    "print(\"Label\", \"r\", \"c\", sep=\"\\t\")\n",
    "print(\"-\" * 20)\n",
    "for k, v in sorted(langerin_reduced.labels.info.params.items()):\n",
    "    print(k, *v, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Freeze this cluster result\n",
    "langerin_reduced.isolate()\n",
    "current = c1 = langerin_reduced.children[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "By default, the isolation assigns an alias to each child cluster that reveals its origin. In this case \"root - 1\", translates to the first child cluster of the root data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now cluster the first child cluster from this current cluster result with increased density criteria."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.2149 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "18839     0.400     10        10        None      3         0.946     0.029     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29472 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.4\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 10, member_cutoff=10)\n",
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Good ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.2091 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "18839     0.375     10        10        None      5         0.825     0.041     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 34900 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.375\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 10, member_cutoff=10)\n",
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Better ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.1514 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "18839     0.300     10        10        None      5         0.774     0.114     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 38192 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.3\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 10, member_cutoff=10)\n",
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Probably to much! Let's keep the second last try."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also work on the second cluster of the first split."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "current = c2 = langerin_reduced.children[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.0485 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "6993      0.400     5         10        None      2         0.615     0.013     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 19880 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recluster second cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.4\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 5, member_cutoff=10)\n",
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The re-clustered data points still may be clustered further."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "current = c1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label\tr\tc\n",
      "--------------------\n",
      "1\t0.375\t10\n",
      "2\t0.375\t10\n",
      "3\t0.375\t10\n",
      "4\t0.375\t10\n",
      "5\t0.375\t10\n"
     ]
    }
   ],
   "source": [
    "# Lets make sure we work on the correctly clustered object\n",
    "print(\"Label\", \"r\", \"c\", sep=\"\\t\")\n",
    "print(\"-\" * 20)\n",
    "for k, v in sorted(current.labels.info.params.items()):\n",
    "    print(k, *v, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "current.isolate()\n",
    "current = c1_1 = current.children[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CNN clustering object (root - 1 - 1)"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "current"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-15T10:22:21.659782Z",
     "start_time": "2020-05-15T10:22:06.533636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 14.5640 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "15541     0.280     15        10        None      2         0.676     0.142     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 25276 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.28\n",
    "# current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 15, member_cutoff=10)\n",
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering hierarchy level 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And on it goes ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "current.isolate()\n",
    "current = c1_1_1 = current.children[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time for call of fit: 0 hours, 0 minutes, 0.0792 seconds\n",
      "--------------------------------------------------------------------------------\n",
      "#points   R         C         min       max       #clusters %largest  %noise    \n",
      "10506     0.220     15        10        None      2         0.650     0.309     \n",
      "--------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 20500 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Recluster first cluster from last step\n",
    "current.data.points.cKDTree()  # We pre-compute neighbourhoods here\n",
    "r = 0.22\n",
    "current.calc_neighbours_from_cKDTree(r=r)\n",
    "current.fit(r, 15, member_cutoff=10)\n",
    "draw_evaluate(current)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge hierarchy levels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to leave it with that for the moment. We can visualise our cluster hierarchy to get an overview. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 95276 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Cluster hierarchy as pie diagram (hierarchy level increasing from the center outwards)\n",
    "_ = langerin_reduced.pie()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we can finally put everything together and incorporate the child clusters into the root data set. We get an impression on how big the extracted clusters are relative to each other. In principle we have 5 or 6 dominating clusters in terms of size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deep=None goes done to the bottom of the hierarchy\n",
    "langerin_reduced.reel(deep=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After this call, cluster labeling may not be contiguous and sorted by size, which we can fix easily."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.labels.sort_by_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 68920 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label    Size\n",
      "=============\n",
      "(0, 6617)\n",
      "(1, 6831)\n",
      "(2, 4298)\n",
      "(3, 2834)\n",
      "(4, 2604)\n",
      "(5, 2112)\n",
      "(6, 425)\n",
      "(7, 324)\n",
      "(8, 191)\n",
      "(9, 72)\n",
      "(10, 56)\n",
      "(11, 44)\n",
      "(12, 42)\n",
      "(13, 40)\n",
      "(14, 38)\n"
     ]
    }
   ],
   "source": [
    "print(\"Label    Size\")\n",
    "print(\"=============\")\n",
    "print(*sorted({k: len(v) for k, v in langerin_reduced.labels.clusterdict.items()}.items()), sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For later re-use, we save the clustering result in form of the cluster label assignments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save\n",
    "np.save(\"md_example/cluster_labels.npy\", langerin_reduced.labels)\n",
    "with open(\"md_example/cluster_labels_info.json\", \"w\") as f_:\n",
    "    f_.write(json.dumps(langerin_reduced.labels.info.params, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read\n",
    "langerin_reduced.labels = np.load(\"md_example/cluster_labels.npy\")\n",
    "with open(\"md_example/cluster_labels_info.json\") as f_:\n",
    "    langerin_reduced.labels.info = cnn.LabelInfo(None, None, json.load(f_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have several option to fine tune our cluster result. Depending on the used kind of validation of the result, clusters may turn out to be irelavant or splitted to much. If we want to throw out a number of clusters for any reason, we can do this by declaring all their membering points as noise:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.labels.trash([6, 7, 10, 11, 12, 13])\n",
    "langerin_reduced.labels.sort_by_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label    Size\n",
      "=============\n",
      "(0, 7548)\n",
      "(1, 6831)\n",
      "(2, 4298)\n",
      "(3, 2834)\n",
      "(4, 2604)\n",
      "(5, 2112)\n",
      "(6, 191)\n",
      "(7, 72)\n",
      "(8, 38)\n"
     ]
    }
   ],
   "source": [
    "print(\"Label    Size\")\n",
    "print(\"=============\")\n",
    "print(*sorted({k: len(v) for k, v in langerin_reduced.labels.clusterdict.items()}.items()), sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 60624 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also group clusters together:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced.labels.merge([7, 8])\n",
    "langerin_reduced.labels.sort_by_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59892 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MSM estimation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assuming our data was sampled in a time-correlated manner as it is the case for MD simulation data, we can use this clustering result as a basis for the estimation of a core-set Markov-state model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = cmsm.CMSM(langerin_reduced.get_dtraj(), unit=\"ns\", step=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Estimate csMSM for different lag times (given in steps)\n",
    "lags = [1, 2, 4, 8, 15, 30]\n",
    "for i in lags:\n",
    "    M.cmsm(lag=i, minlenfactor=5, v=False)\n",
    "    M.get_its()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.0, 10459.606311240232)]"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 37780 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the time scales\n",
    "fig, ax, *_ = M.plot_its()\n",
    "fig.tight_layout(pad=0.1)\n",
    "ax.set(**{\n",
    "    \"ylim\": (0, None)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13572 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "matrix = ax.imshow(M.T, cmap=mpl.cm.inferno)\n",
    "fig.colorbar(matrix)\n",
    "ax.set(**{\n",
    "    \"aspect\": \"equal\",\n",
    "    \"xticks\": range(len(M.T)),\n",
    "    \"xticklabels\": range(1, len(M.T) + 1),\n",
    "    \"yticks\": range(len(M.T)),\n",
    "    \"yticklabels\": range(1, len(M.T) + 1)\n",
    "})\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label\tr\tc\n",
      "--------------------\n",
      "1\t0.19\t15\n",
      "2\t0.4\t5\n",
      "3\t0.25\t15\n",
      "4\t0.4\t5\n",
      "5\t0.375\t10\n",
      "6\t0.375\t10\n",
      "7\t0.19\t15\n",
      "8\t0.19\t15\n",
      "9\t0.5\t5\n",
      "10\t0.5\t5\n",
      "11\t0.5\t5\n",
      "12\t0.5\t5\n",
      "13\t0.375\t10\n",
      "14\t0.375\t10\n",
      "15\t0.5\t5\n",
      "16\t0.19\t15\n",
      "17\t0.25\t15\n"
     ]
    }
   ],
   "source": [
    "# Lets make sure we work on the correctly clustered object\n",
    "print(\"Label\", \"r\", \"c\", sep=\"\\t\")\n",
    "print(\"-\" * 20)\n",
    "for k, v in sorted(langerin_reduced.labels.info.params.items()):\n",
    "    print(k, *v, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "langerin_reduced_less = langerin.cut(points=(None, None, 50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 10.5 GiB for an array with shape (52942, 26528) and data type float64",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-143-4379ad10935c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlangerin_reduced_less\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcalc_dist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlangerin_reduced\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmmap\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmmap_file\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"/home/janjoswig/tmp/tmp.npy\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# Distance map calculation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~/CNN/cnnclustering/cnn.py\u001b[0m in \u001b[0;36mcalc_dist\u001b[1;34m(self, other, v, method, mmap, mmap_file, chunksize, progress, **kwargs)\u001b[0m\n\u001b[0;32m   1732\u001b[0m                 \u001b[0mlen_self\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpoints\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1733\u001b[0m                 \u001b[0mlen_other\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpoints\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1734\u001b[1;33m                 self.data.distances = np.memmap(\n\u001b[0m\u001b[0;32m   1735\u001b[0m                     \u001b[0mmmap_file\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1736\u001b[0m                     dtype=settings.float_precision_map[\n",
      "\u001b[1;32m~/CNN/cnnclustering/cnn.py\u001b[0m in \u001b[0;36mdistances\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m   1124\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mdistances\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1125\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDistances\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1126\u001b[1;33m             \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDistances\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1127\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_distances\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1128\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~/CNN/cnnclustering/cnn.py\u001b[0m in \u001b[0;36m__new__\u001b[1;34m(cls, d, reference)\u001b[0m\n\u001b[0;32m    722\u001b[0m             \u001b[0md\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    723\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 724\u001b[1;33m         \u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mview\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    725\u001b[0m         \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reference\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    726\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~/.pyenv/versions/miniconda3-4.7.12/envs/cnnclustering/lib/python3.8/site-packages/numpy/core/_asarray.py\u001b[0m in \u001b[0;36masarray\u001b[1;34m(a, dtype, order)\u001b[0m\n\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \"\"\"\n\u001b[1;32m---> 83\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 10.5 GiB for an array with shape (52942, 26528) and data type float64"
     ]
    }
   ],
   "source": [
    "langerin_reduced_less.calc_dist(langerin_reduced, mmap=True, mmap_file=\"/home/janjoswig/tmp/tmp.npy\", chunksize=1000)  # Distance map calculation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster alternatives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is always recommended to cross validate a clustering result with the outcome of other clustering approaches. We want to have a quick look at the alternative that density-peak clustering provides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32548 characters]\n",
      "text/plain": [
       "<Figure size 750x750 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pydpc_clustering = pydpc.Cluster(langerin_reduced.data.points)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clustering in this case is just a single step without the need of parameter specification. In the following, however, we need to extract the actual clusters by looking at the plot below. Points that are clearly isolated in this plot are highly reliable cluster centers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "pydpc_clustering.autoplot = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32476 characters]\n",
      "text/plain": [
       "<Figure size 750x750 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pydpc_clustering.assign(0, 1.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This gives us 7 clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41768 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin_reduced.labels = (pydpc_clustering.membership + 1)\n",
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we are interested in core clusters we want to apply the core/halo criterion to disregard points with low cluster membership probabilitie as noise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58248 characters]\n",
      "text/plain": [
       "<Figure size 750x225 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "langerin_reduced.labels[pydpc_clustering.halo_idx] = 0\n",
    "draw_evaluate(langerin_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = cmsm.CMSM(langerin_reduced.get_dtraj(), unit=\"ns\", step=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T09:31:02.428514Z",
     "start_time": "2019-09-12T09:31:02.357788Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 1 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 2 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 4 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 8 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Using 116 trajectories with 25900 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 15 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Trajectories [0, 1, 73]\n",
      "are shorter then step threshold (lag * minlenfactor = 75)\n",
      "and will not be used to compute the MSM.\n",
      "\n",
      "Using 113 trajectories with 25732 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n",
      "\n",
      "*********************************************************\n",
      "---------------------------------------------------------\n",
      "Computing coreset MSM at lagtime 30 ns\n",
      "---------------------------------------------------------\n",
      "\n",
      "Trajectories [0, 1, 4, 63, 73]\n",
      "are shorter then step threshold (lag * minlenfactor = 150)\n",
      "and will not be used to compute the MSM.\n",
      "\n",
      "Using 111 trajectories with 25447 steps over 7 coresets\n",
      "\n",
      "All sets are connected\n",
      "---------------------------------------------------------\n",
      "*********************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Estimate csMSM for different lag times (given in steps)\n",
    "lags = [1, 2, 4, 8, 15, 30]\n",
    "for i in lags:\n",
    "    M.cmsm(lag=i, minlenfactor=5)\n",
    "    M.get_its()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T09:31:20.706043Z",
     "start_time": "2019-09-12T09:31:19.851168Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45388 characters]\n",
      "text/plain": [
       "<Figure size 750x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the time scales\n",
    "fig, ax, *_ = M.plot_its()\n",
    "fig.tight_layout(pad=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56724 characters]\n",
      "text/plain": [
       "<Figure size 750x630 with 7 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize = mpl.rcParams[\"figure.figsize\"]\n",
    "mpl.rcParams[\"figure.figsize\"] = figsize[0], figsize[1] * 0.2\n",
    "M.plot_eigenvectors()\n",
    "mpl.rcParams[\"figure.figsize\"] = figsize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This result is in good agreement with the one we obtained manually and argualbe faster and easier to achieve. If we decide that this result is exactly what we consider valid, then this is nice. If we on the other hand want to tune the clustering result further, with respect to splitting, noise level and what is considered noise in the first place we gain more flexibility with the manual approach."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cnnclustering38",
   "language": "python",
   "name": "cnnclustering38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "164.988px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "toc-autonumbering": false,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false,
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
