{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "import qp\n",
    "import qp.metrics\n",
    "\n",
    "\"\"\"\n",
    "When producing random variates from distributions contained in an Ensemble object\n",
    "rvs_size specifies the number of variates to produce, and random_state ensures\n",
    "reproducibility for this notebook.\n",
    "\"\"\"\n",
    "rvs_size = 100\n",
    "random_state = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following line produces an Ensemble object that contains 11 normal distributions. The mean and sigma are different for each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_distributions = 3\n",
    "\n",
    "locs_1 = 2* (np.random.uniform(size=(number_of_distributions,1)))\n",
    "scales_1 = 1 + 0.2*(np.random.uniform(size=(number_of_distributions,1)))\n",
    "ens_r_1 = qp.Ensemble(qp.stats.rayleigh, data=dict(loc=locs_1, scale=scales_1))\n",
    "\n",
    "locs_2 = 2* (np.random.uniform(size=(number_of_distributions,1)))\n",
    "scales_2 = 1 + 0.2*(np.random.uniform(size=(number_of_distributions,1)))\n",
    "ens_r_2 = qp.Ensemble(qp.stats.rayleigh, data=dict(loc=locs_2, scale=scales_2))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the Kolmogorov-Smirnov statistic between each pair of distributions in two Ensembles. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_2,\n",
    "            fit_metric='ks', # 'ks' = Kolmogorov-Smirnov. 'ad' for Anderson-Darling and 'cvm' for Cramer-von Mises are also available.\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the PDFs along with the resulting Kolmogorov-Smirnov statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax0, ax1, ax2) = plt.subplots(3, 1, sharex=True)\n",
    "ax0.set_xlim([0,8])\n",
    "\n",
    "ax0.text(0.95, 0.5, f'KS stat = {output[0]}', verticalalignment='bottom', horizontalalignment='right', transform=ax0.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[0], axes=ax0, color='black')\n",
    "_ = qp.plotting.plot_native(ens_r_2[0], axes=ax0, color='black', linestyle='dashed')\n",
    "\n",
    "ax1.text(0.95, 0.5, f'KS stat = {output[1]}', verticalalignment='bottom', horizontalalignment='right', transform=ax1.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[1], axes=ax1, color='black')\n",
    "_ = qp.plotting.plot_native(ens_r_2[1], axes=ax1, color='black', linestyle='dashed')\n",
    "\n",
    "ax2.text(0.95, 0.5, f'KS stat = {output[2]}', verticalalignment='bottom', horizontalalignment='right', transform=ax2.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[2], axes=ax2, color='black')\n",
    "_ = qp.plotting.plot_native(ens_r_2[2], axes=ax2, color='black', linestyle='dashed')\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly Anderson-Darling and Cramer-von Mises statistics can be calculated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_output = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_2,\n",
    "            fit_metric='ad', # 'ad' = Anderson-Darling\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "cvm_output = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_2,\n",
    "            fit_metric='cvm', # 'cvm' = Cramer-von Mises\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax0, ax1, ax2) = plt.subplots(3, 1, sharex=True)\n",
    "ax0.set_xlim([0,8])\n",
    "\n",
    "ax0.text(0.95, 0.65, f'AD stat = {ad_output[0]}', verticalalignment='bottom', horizontalalignment='right', transform=ax0.transAxes)\n",
    "ax0.text(0.95, 0.5, f'CvM stat = {cvm_output[0]}', verticalalignment='bottom', horizontalalignment='right', transform=ax0.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[0], axes=ax0, color='black')\n",
    "_ = qp.plotting.plot_native(ens_r_2[0], axes=ax0, color='black', linestyle='dashed')\n",
    "\n",
    "ax1.text(0.95, 0.65, f'AD stat = {ad_output[1]}', verticalalignment='bottom', horizontalalignment='right', transform=ax1.transAxes)\n",
    "ax1.text(0.95, 0.5, f'CvM stat = {cvm_output[1]}', verticalalignment='bottom', horizontalalignment='right', transform=ax1.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[1], axes=ax1, color='black')\n",
    "_ = qp.plotting.plot_native(ens_r_2[1], axes=ax1, color='black', linestyle='dashed')\n",
    "\n",
    "ax2.text(0.95, 0.65, f'AD stat = {ad_output[2]}', verticalalignment='bottom', horizontalalignment='right', transform=ax2.transAxes)\n",
    "ax2.text(0.95, 0.5, f'CvM stat = {cvm_output[2]}', verticalalignment='bottom', horizontalalignment='right', transform=ax2.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[2], axes=ax2, color='black')\n",
    "_ = qp.plotting.plot_native(ens_r_2[2], axes=ax2, color='black', linestyle='dashed')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition to Ensemble vs.0 Ensemble pair-wise calculations, we can compare an Ensemble containing N distributions against an Ensemble containing 1 distribution. We'll create an Ensemble containing a single distribution, and then calculate the various statistics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "locs = 2* (np.random.uniform(size=(1,1)))\n",
    "scales = 1 + 0.2*(np.random.uniform(size=(1,1)))\n",
    "ens_r_single = qp.Ensemble(qp.stats.rayleigh, data=dict(loc=locs, scale=scales))\n",
    "\n",
    "print(\"Number of distributions in ens_r_2: \" + str(ens_r_single.npdf))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that when comparing Ensemble(N) vs. Ensemble(1), the Ensemble containing 1 distribution should be the second argument passed to the function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_output_vs_1 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_single,\n",
    "            fit_metric='ad',\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "cvm_output_vs_1 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_single,\n",
    "            fit_metric='cvm',\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "ks_output_vs_1 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_single,\n",
    "            fit_metric='ks',\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax0, ax1, ax2) = plt.subplots(3, 1, sharex=True)\n",
    "ax0.set_xlim([0,8])\n",
    "\n",
    "ax0.text(0.95, 0.5, f'AD stat = {ad_output_vs_1[0]}', verticalalignment='bottom', horizontalalignment='right', transform=ax0.transAxes)\n",
    "ax0.text(0.95, 0.65, f'CvM stat = {cvm_output_vs_1[0]}', verticalalignment='bottom', horizontalalignment='right', transform=ax0.transAxes)\n",
    "ax0.text(0.95, 0.8, f'KS stat = {ks_output_vs_1[0]}', verticalalignment='bottom', horizontalalignment='right', transform=ax0.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[0], axes=ax0, color='black', linestyle='dashed')\n",
    "_ = qp.plotting.plot_native(ens_r_single, axes=ax0, color='black')\n",
    "\n",
    "ax1.text(0.95, 0.5, f'AD stat = {ad_output_vs_1[1]}', verticalalignment='bottom', horizontalalignment='right', transform=ax1.transAxes)\n",
    "ax1.text(0.95, 0.65, f'CvM stat = {cvm_output_vs_1[1]}', verticalalignment='bottom', horizontalalignment='right', transform=ax1.transAxes)\n",
    "ax1.text(0.95, 0.8, f'KS stat = {ks_output_vs_1[1]}', verticalalignment='bottom', horizontalalignment='right', transform=ax1.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[1], axes=ax1, color='black', linestyle='dashed')\n",
    "_ = qp.plotting.plot_native(ens_r_single, axes=ax1, color='black')\n",
    "\n",
    "ax2.text(0.95, 0.5, f'AD stat = {ad_output_vs_1[2]}', verticalalignment='bottom', horizontalalignment='right', transform=ax2.transAxes)\n",
    "ax2.text(0.95, 0.65, f'CvM stat = {cvm_output_vs_1[2]}', verticalalignment='bottom', horizontalalignment='right', transform=ax2.transAxes)\n",
    "ax2.text(0.95, 0.8, f'KS stat = {ks_output_vs_1[2]}', verticalalignment='bottom', horizontalalignment='right', transform=ax2.transAxes)\n",
    "_ = qp.plotting.plot_native(ens_r_1[2], axes=ax2, color='black', linestyle='dashed')\n",
    "_ = qp.plotting.plot_native(ens_r_single, axes=ax2, color='black')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that because we are sampling from distributions in one Ensemble, the calculation is not symmetric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_1_then_2 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_2,\n",
    "            fit_metric='ad',\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "\n",
    "# Here, the order of the input variables has changed from ens_r_1, ens_r_2 to ens_r_2, ens_r_1\n",
    "output_2_then_1 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_2,\n",
    "            ens_r_1,\n",
    "            fit_metric='ad',\n",
    "            num_samples=rvs_size,\n",
    "            _random_state=random_state\n",
    "        )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If the calculation were symmetric, we would expect the difference between the two outputs to be approximately [0,0,0]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(output_1_then_2 - output_2_then_1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This also implies that, because we sample from the distributions, the number of random variates requested will affect the output. \n",
    "Below we compare distributions of a single Ensemble pair-wise against themselves. As we increase the value for `num_samples` the resulting Kolmogorov-Smirnov value trends toward 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_1 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_1,\n",
    "            fit_metric='ks',\n",
    "            num_samples=1,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "output_10 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_1,\n",
    "            fit_metric='ks',\n",
    "            num_samples=10,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "output_100 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_1,\n",
    "            fit_metric='ks',\n",
    "            num_samples=100,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "output_1000 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_1,\n",
    "            fit_metric='ks',\n",
    "            num_samples=1_000,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "output_10000 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_1,\n",
    "            fit_metric='ks',\n",
    "            num_samples=10_000,\n",
    "            _random_state=random_state\n",
    "        )\n",
    "\n",
    "output_100000 = qp.metrics.calculate_goodness_of_fit(\n",
    "            ens_r_1,\n",
    "            ens_r_1,\n",
    "            fit_metric='ks',\n",
    "            num_samples=100_000,\n",
    "            _random_state=random_state\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['1', '2', '3']\n",
    "fig, ax = plt.subplots()\n",
    "x = np.arange(len(labels))\n",
    "width = 0.5/6\n",
    "ax.bar(x + width*0, output_1, width, label='1 Sample')\n",
    "ax.bar(x + width*1, output_10, width, label='10 Samples')\n",
    "ax.bar(x + width*2, output_100, width, label='100 Samples')\n",
    "ax.bar(x + width*3, output_1000, width, label='1000 Sample')\n",
    "ax.bar(x + width*4, output_10000, width, label='10000 Sample')\n",
    "ax.bar(x + width*5, output_100000, width, label='100000 Samples')\n",
    "\n",
    "fig.tight_layout()\n",
    "ax.set_title('Pair-wise Comparison of Identical Distributions for Increasing Number of Samples')\n",
    "ax.set_xticks(x, labels)\n",
    "ax.set_xlabel('Example distribution in Ensemble')\n",
    "ax.set_ylabel('Kolmogorov-Smirnov Statistic')\n",
    "ax.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('qp_issue_25')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "9be1a7334e581107f2753dc5e5ebb12b8975e5f80325d35375311bd0b6b51792"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
