{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!-- Auto table of contents -->\n",
    "<h1 class='tocIgnore'>Development archives (low level code)</h1>\n",
    "\n",
    "**pyaudisam**: Automation of Distance Sampling analyses with [Distance software](http://distancesampling.org/)\n",
    "\n",
    "Copyright (C) 2021 Jean-Philippe Meuret\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify it under the terms\n",
    "of the GNU General Public License as published by the Free Software Foundation,\n",
    "either version 3 of the License, or (at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n",
    "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n",
    "See the GNU General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU General Public License along with this program.\n",
    "If not, see https://www.gnu.org/licenses/.\n",
    "\n",
    "<div style=\"overflow-y: auto\">\n",
    "  <h2 class='tocIgnore'>Table of contents</h2>\n",
    "  <div id=\"toc\"></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "$.getScript('ipython_notebook_toc.js')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import pathlib as pl\n",
    "\n",
    "import re\n",
    "\n",
    "import concurrent.futures as cofu\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activate Warnings as Exception\n",
    "#import warnings\n",
    "#warnings.filterwarnings('error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyaudisam as ads\n",
    "\n",
    "ads.runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create temporary directory if not yet done.\n",
    "tmpDir = pl.Path('tmp')\n",
    "tmpDir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logging configuration.\n",
    "ads.log.configure(handlers=[sys.stdout, tmpDir / 'devarc1.log'], verbose=True, reset=True)\n",
    "\n",
    "ads.logger('matplotlib', level=ads.WARNING, reset=True)\n",
    "\n",
    "ads.logger('ads', level=ads.INFO, reset=True)\n",
    "ads.logger('ads.dat', level=ads.INFO, reset=True)\n",
    "ads.logger('ads.eng', level=ads.INFO2, reset=True)\n",
    "ads.logger('ads.opn', level=ads.INFO1, reset=True)\n",
    "ads.logger('ads.opr', level=ads.INFO1, reset=True)\n",
    "ads.logger('ads.anr', level=ads.INFO1, reset=True)\n",
    "\n",
    "logger = ads.logger('unintst', level=ads.DEBUG, reset=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Development of pyaudisam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distance detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distance software detection params.\n",
    "DistanceSuppVers = [7, 6] # Lastest first.\n",
    "DistancePossInstPaths = [pl.Path().resolve(), pl.Path('C:\\\\Program files (x86)'),\n",
    "                         pl.Path('C:\\\\Program files'), pl.Path('C:\\\\PortableApps')]\n",
    "\n",
    "# Find given executable installation dir.\n",
    "def findExecutable(exeFileName):\n",
    "\n",
    "    exeFilePathName = None\n",
    "    print('Looking for {} ...'.format(exeFileName))\n",
    "    for path in DistancePossInstPaths:\n",
    "        for ver in DistanceSuppVers:\n",
    "            exeFileDir = path / 'Distance {}'.format(ver)\n",
    "            print(' - checking {} : '.format(exeFileDir), end='')\n",
    "            exeFN = exeFileDir / exeFileName\n",
    "            if not exeFN.exists():\n",
    "                print('no.')\n",
    "            else:\n",
    "                print('yes !')\n",
    "                exeFilePathName = exeFN\n",
    "                break\n",
    "        if exeFilePathName:\n",
    "            break\n",
    "\n",
    "    if exeFilePathName:\n",
    "        print('{} found in {}'.format(exeFileName, exeFileDir))\n",
    "    else:\n",
    "        raise Exception('Could not find {} ; please install Distance software (V6 or later)'.format(exeFileName))\n",
    "\n",
    "    return exeFilePathName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "findExecutable('MCDS.exe')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results reports styling\n",
    "\n",
    "(to stress interesting and/or important things)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTrSynRes = results.dfTransData('fr', columns=synthCols)\n",
    "dfTrSynRes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cChrGray = '#869074'\n",
    "cBckGreen, cBckGray = '#e0ef8c', '#dae3cb'\n",
    "cSclGreen, cSclOrange, cSclRed = '#cbef8c', '#f9da56', '#fe835a' #'#f25e2d'\n",
    "scaledColors = [cSclGreen, cSclOrange, cSclRed]\n",
    "scaledColorsRvd = list(reversed(scaledColors))\n",
    "\n",
    "dExCodeColors = dict(zip([1, 2, 3], scaledColors))\n",
    "def colorExecCodes(sCodes):\n",
    "    return ['background-color: ' + dExCodeColors.get(c, dExCodeColors[3]) for c in sCodes]\n",
    "\n",
    "def scaledColorV(v, thresholds, colors): # len(thresholds) == len(colors) - 1\n",
    "    if pd.isnull(v):\n",
    "        return cBckGray\n",
    "    for ind, thresh in enumerate(thresholds):\n",
    "        if v > thresh:\n",
    "            return colors[ind]\n",
    "    return colors[-1]\n",
    "def scaledColorS(sValues, thresholds, colors):\n",
    "    return ['background-color: ' + scaledColorV(v, thresholds, colors) for v in sValues]\n",
    "\n",
    "densCVThresholds = [0.4, 0.1]\n",
    "\n",
    "dfs = dfTrSynRes \\\n",
    "        .sort_values(by=['Espèce', 'Echantillon', 'Précision', 'Durée', 'Delta AIC']) \\\n",
    "        .style \\\n",
    "        .set_precision(3) \\\n",
    "        .set_properties(subset=pd.IndexSlice[dfTrSynRes[dfTrSynRes['Delta AIC'] == 0].index, :],\n",
    "                        **{'background-color': cBckGreen}) \\\n",
    "        .apply(colorExecCodes, subset=['CodEx'], axis='columns') \\\n",
    "        .apply(scaledColorS, subset=['CoefVar Densité'], axis='columns',\n",
    "               thresholds=densCVThresholds, colors=scaledColors) \\\n",
    "        .set_properties(subset=pd.IndexSlice[dfTrSynRes[~dfTrSynRes.CodEx.isin([1, 2])].index, :],\n",
    "                         **{'color': cChrGray}) \\\n",
    "        .where(pd.isnull, 'color: transparent')\n",
    "\n",
    "    #.format(lambda v: v if not pd.isnull(v) else '') # Détruit une partie des arrondis, auugmente la précision ???\n",
    "\n",
    "    #.set_precision(3) # Not really usable, as only for the whole frame\n",
    "\n",
    "    #.apply(lambda s: ['color: grey']*len(s), subset=pd.IndexSlice[dfTrSynRes[~dfTrSynRes.CodEx.isin([1, 2])].index, :],\n",
    "    #       axis='index') # OK\n",
    "    \n",
    "    #.apply(lambda s: ['color: grey']*len(s), subset=dfTrSynRes[~dfTrSynRes.CodEx.isin([1, 2])].index,\n",
    "    #       axis='index') # KO\n",
    "    \n",
    "dfs.to_excel('tmp/styled-results.xlsx')\n",
    "\n",
    "dfs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decode MCDS plots file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import plotly.graph_objs as plygo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "realRunWorkDir = pl.Path('../donnees/acdc/210118-1904/SylvAtri-b-5mn-m-haz-cos-xk2syfzw')\n",
    "[fpn.name for fpn in realRunWorkDir.iterdir()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "srcFileName = pl.Path(realRunWorkDir, 'plots.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = open(srcFileName, 'r').readlines()\n",
    "lines = [line.strip() for line in lines]\n",
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "itLines = iter(lines)\n",
    "chapters = list()\n",
    "for title in itLines:\n",
    "    #title = next(itLines)\n",
    "    subTitle = next(itLines)\n",
    "    xLabel = next(itLines)\n",
    "    yLabel = next(itLines)\n",
    "    xMin, xMax, yMin, yMax = [float(s) for s in next(itLines).split()]\n",
    "    nDataRows = int(next(itLines))\n",
    "    dataRows = list()\n",
    "    for l in range(nDataRows):\n",
    "        dataRows.append([float(s) for s in next(itLines).split()])\n",
    "    chapters.append(dict(title=title, subTitle=subTitle, dataRows=dataRows, #nDataRows=nDataRows,\n",
    "                         xLabel=xLabel, yLabel=yLabel, xMin=xMin, xMax=xMax, yMin=yMin, yMax=yMax))\n",
    "len(chapters), chapters[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## QQ-plot\n",
    "chapter = chapters[0]\n",
    "chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = len(chapter['dataRows'])\n",
    "dfQqData = pd.DataFrame(data=chapter['dataRows'], columns=['If the fit was perfect ...', 'Real observations'],\n",
    "                        index=np.linspace(0.5/n, 1.0-0.5/n, n))\n",
    "dfQqData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Option 1 : OK\n",
    "#fig = plt.figure(figsize=(16, 6))\n",
    "#axes = fig.subplots()\n",
    "#_ = dfQqData.plot(ax=axes, color=['blue', 'red'], grid=True,\n",
    "#                  xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))\n",
    "\n",
    "# Option 2 : OK\n",
    "axes = dfQqData.plot(figsize=(16, 6), color=['blue', 'red'], grid=True,\n",
    "                     \n",
    "                     xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))\n",
    "fig = axes.figure\n",
    "\n",
    "axes.legend(['If the fit was perfect ...', 'Real observations'], fontsize=12)\n",
    "axes.set_facecolor('#f9fbf3')\n",
    "axes.figure.patch.set_facecolor('#f9fbf3')\n",
    "axes.set_title(label=chapter['title'] + ' : ' + chapter['subTitle'], fontdict=dict(fontsize=16), pad=20)\n",
    "axes.set_xlabel(chapter['xLabel'], fontsize=12)\n",
    "_ = axes.set_ylabel(chapter['yLabel'], fontsize=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "axes.figure.savefig('tmp/mlb-qqplot.jpg', box_inches='tight')\n",
    "axes.figure.savefig('tmp/mlb-qqplot.png', box_inches='tight', transparent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Detection probability 1\n",
    "chapter = chapters[1]\n",
    "chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfDetProbData = pd.DataFrame(data=chapter['dataRows'], \n",
    "                             columns=[chapter['xLabel'], chapter['yLabel'] + ' (sampled)', chapter['yLabel'] + ' (fitted)'])\n",
    "dfDetProbData.set_index(chapter['xLabel'], inplace=True)\n",
    "dfDetProbData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "axes = dfDetProbData.plot(figsize=(16, 6), color=['blue', 'red'], grid=True,\n",
    "                          xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))\n",
    "\n",
    "axes.set_title(label=chapter['title'] + ' : ' + chapter['subTitle'], fontdict=dict(fontsize=16), pad=20)\n",
    "axes.legend(dfDetProbData.columns, fontsize=12)\n",
    "axes.set_xlabel(chapter['xLabel'], fontsize=12)\n",
    "_ = axes.set_ylabel(chapter['yLabel'], fontsize=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plotly 4\n",
    "fig = plygo.Figure()\n",
    "\n",
    "fig.add_trace(plygo.Scatter(x=dfQqData.index, y=dfQqData['If the fit was perfect ...'],\n",
    "                            name='If the fit was perfect ...', line=dict(color='blue', width=2), opacity=0.7))\n",
    "fig.add_trace(plygo.Scatter(x=dfQqData.index, y=dfQqData['Real observations'],\n",
    "                            name='Real observations', line=dict(color='red', width=2)))\n",
    "\n",
    "fig.update_layout(title=chapter['title'] + ' : ' + chapter['subTitle'],\n",
    "                  xaxis=dict(title=chapter['xLabel'], range=(chapter['xMin'], chapter['xMax']),\n",
    "                             zeroline=True, linewidth=1, linecolor='black'),\n",
    "                  yaxis=dict(title=chapter['yLabel'], range=(chapter['yMin'], chapter['yMax']),\n",
    "                             zeroline=True, linewidth=1, linecolor='black'),\n",
    "                  legend=plygo.layout.Legend(x=0.09, y=0.90, bordercolor='black', borderwidth=1),\n",
    "                  shapes=[plygo.layout.Shape(type='line', x0=chapter['xMax'], y0=chapter['yMin'],\n",
    "                                                          x1=chapter['xMax'], y1=chapter['yMax']),\n",
    "                          plygo.layout.Shape(type='line', x0=chapter['xMin'], y0=chapter['yMax'],\n",
    "                                                          x1=chapter['xMax'], y1=chapter['yMax'])],\n",
    "                  template='none')\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wow ... VERY slooooooooow !\n",
    "fig.write_image(\"tmp/ply-qqplot.svg\")\n",
    "fig.write_image(\"tmp/ply-qqplot.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plotly 4\n",
    "fig = plygo.Figure()\n",
    "\n",
    "fig.add_trace(plygo.Scatter(x=dfDetProbData.index, y=dfDetProbData[chapter['yLabel'] + ' (sampled)'],\n",
    "                            name=chapter['yLabel'] + ' (sampled)', line=dict(color='blue', width=2), opacity=0.7))\n",
    "fig.add_trace(plygo.Scatter(x=dfDetProbData.index, y=dfDetProbData[chapter['yLabel'] + ' (fitted)'],\n",
    "                            name=chapter['yLabel'] + ' (fitted)', line=dict(color='red', width=2)))\n",
    "\n",
    "fig.update_layout(title=chapter['title'] + ' : ' + chapter['subTitle'],\n",
    "                  xaxis=dict(title=chapter['xLabel'], range=(chapter['xMin'], chapter['xMax']),\n",
    "                             zeroline=True, linewidth=1, linecolor='black'),\n",
    "                  yaxis=dict(title=chapter['yLabel'], range=(chapter['yMin'], chapter['yMax']),\n",
    "                             zeroline=True, linewidth=1, linecolor='black'),\n",
    "                  legend=plygo.layout.Legend(x=0.65, y=0.85*chapter['yMax'], bordercolor='black', borderwidth=1),\n",
    "                  shapes=[plygo.layout.Shape(type='line', x0=chapter['xMax'], y0=chapter['yMin'],\n",
    "                                                          x1=chapter['xMax'], y1=chapter['yMax']),\n",
    "                          plygo.layout.Shape(type='line', x0=chapter['xMin'], y0=chapter['yMax'],\n",
    "                                                          x1=chapter['xMax'], y1=chapter['yMax'])],\n",
    "                  template='none')\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PDF 1\n",
    "chapter = chapters[2]\n",
    "chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfProdDensData = pd.DataFrame(data=chapter['dataRows'], \n",
    "                              columns=[chapter['xLabel'], chapter['yLabel'] + ' (sampled)', chapter['yLabel'] + ' (fitted)'])\n",
    "dfProdDensData.set_index(chapter['xLabel'], inplace=True)\n",
    "dfProdDensData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "axes = dfProdDensData.plot(figsize=(16, 6), color=['blue', 'red'],\n",
    "                           xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))\n",
    "axes.set_title(label=chapter['title'] + ' : ' + chapter['subTitle'], fontdict=dict(fontsize=16), pad=20)\n",
    "axes.legend(dfProdDensData.columns, fontsize=12)\n",
    "axes.set_xlabel(chapter['xLabel'], fontsize=12)\n",
    "_ = axes.set_ylabel(chapter['yLabel'], fontsize=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plotly 4\n",
    "fig = plygo.Figure()\n",
    "\n",
    "fig.add_trace(plygo.Scatter(x=dfProdDensData.index, y=dfProdDensData[chapter['yLabel'] + ' (sampled)'],\n",
    "                            name=chapter['yLabel'] + ' (sampled)', line=dict(color='blue', width=2), opacity=0.7))\n",
    "fig.add_trace(plygo.Scatter(x=dfProdDensData.index, y=dfProdDensData[chapter['yLabel'] + ' (fitted)'],\n",
    "                            name=chapter['yLabel'] + ' (fitted)', line=dict(color='red', width=2)))\n",
    "\n",
    "fig.update_layout(title=chapter['title'] + ' : ' + chapter['subTitle'],\n",
    "                  xaxis=dict(title=chapter['xLabel'], range=(chapter['xMin'], chapter['xMax']),\n",
    "                             zeroline=True, linewidth=1, linecolor='black'),\n",
    "                  yaxis=dict(title=chapter['yLabel'], range=(chapter['yMin'], chapter['yMax']),\n",
    "                             zeroline=True, linewidth=1, linecolor='black'),\n",
    "                  legend=plygo.layout.Legend(xanchor='right', yanchor='top', bordercolor='black', borderwidth=1),\n",
    "                  #margin=plygo.layout.Margin(l=40, r=40, b=40, t=40, pad=0),\n",
    "                  shapes=[plygo.layout.Shape(type='line', x0=chapter['xMax'], y0=chapter['yMin'],\n",
    "                                                          x1=chapter['xMax'], y1=chapter['yMax']),\n",
    "                          plygo.layout.Shape(type='line', x0=chapter['xMin'], y0=chapter['yMax'],\n",
    "                                                          x1=chapter['xMax'], y1=chapter['yMax'])],\n",
    "                  template='none')\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PDF 3, with stripplot\n",
    "chapter = chapters[6]\n",
    "chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfProdDensData = pd.DataFrame(data=chapter['dataRows'], \n",
    "                              columns=[chapter['xLabel'], chapter['yLabel'] + ' (sampled)', chapter['yLabel'] + ' (fitted)'])\n",
    "dfProdDensData.set_index(chapter['xLabel'], inplace=True)\n",
    "dfProdDensData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(pl.Path(realRunWorkDir, 'cmd.txt'), 'r') as cmdFile:\n",
    "    fieldsLine = next(line for line in cmdFile.readlines() if line.startswith('Fields='))\n",
    "dataCols = fieldsLine.strip('\\n;')[len('Fields='):].split(',')\n",
    "print(dataCols)\n",
    "\n",
    "dfData = pd.read_csv(pl.Path(realRunWorkDir, 'data.txt'), sep='\\t', names=dataCols)\n",
    "sDists = dfData.DISTANCE.dropna()\n",
    "sDists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sb\n",
    "import matplotlib.ticker as pltt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "axes = dfProdDensData.plot(figsize=(16, 6), color=['blue', 'red'], grid=True, linewidth=1,\n",
    "                    xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))\n",
    "axes.set_xlabel(chapter['xLabel'], fontsize=12)\n",
    "axes.set_ylabel(chapter['yLabel'], fontsize=12)\n",
    "axes.set_title(label=chapter['title'] + ' : ' + chapter['subTitle'], fontdict=dict(fontsize=16), pad=20)\n",
    "\n",
    "axes2 = axes.twinx()\n",
    "sb.stripplot(ax=axes2, x=sDists, color='green', size=8, alpha=0.4, jitter=0.3)\n",
    "\n",
    "aMTicks = axes.get_xticks()\n",
    "axes.xaxis.set_minor_locator(pltt.MultipleLocator((aMTicks[1]-aMTicks[0])/5))\n",
    "axes.tick_params(which='minor', grid_linestyle='-.', grid_alpha=0.6)\n",
    "axes.grid(True, which='minor')\n",
    "\n",
    "axes.legend().remove()\n",
    "_ = axes.figure.legend(labels=list(dfProdDensData.columns) + ['Individual observations'], fontsize=12,\n",
    "                       bbox_to_anchor=(1, 1), bbox_transform=axes.transAxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines, labels, lines2, labels2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "axes = dfProdDensData.plot(figsize=(16, 6), color=['blue', 'red'], grid=True,\n",
    "                           xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))\n",
    "axes.set_xlabel(chapter['xLabel'], fontsize=12)\n",
    "axes.set_ylabel(chapter['yLabel'], fontsize=12)\n",
    "axes.set_title(label=chapter['title'] + ' : ' + chapter['subTitle'], fontdict=dict(fontsize=16), pad=20)\n",
    "axes.legend(dfProdDensData.columns, fontsize=12)\n",
    "\n",
    "axes2 = axes.twinx()\n",
    "_ = sb.swarmplot(ax=axes2, x=sDists, color='green', size=8, alpha=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rebuild distance histogram from plot data (intervals) and distance data\n",
    "sh = dfProdDensData['Probability Density (sampled)']\n",
    "bins = [0] + sh.loc[((sh.shift(-1) != sh) | (sh.shift(1) != sh)) & (sh == 0)].index.tolist()\n",
    "bins[-1] += 0.001\n",
    "bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.histogram(sDists, bins=bins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Dropping useless points actually doesn't save overall \"plotting\" time !!!\n",
    "_ = sh.loc[(sh.shift(-1) != sh) | (sh.shift(1) != sh)] \\\n",
    "      .plot(figsize=(16, 6), color='blue', grid=True,\n",
    "            xlim=(chapter['xMin'], chapter['xMax']), ylim=(chapter['yMin'], chapter['yMax']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histogramme à tranches fixes (OK)\n",
    "distBinWidth = 10\n",
    "distBins = np.linspace(start=0, stop=distBinWidth * int(chapter['xMax'] / distBinWidth),\n",
    "                       num=1 + int(chapter['xMax'] / distBinWidth)).tolist()\n",
    "if distBins[-1] < chapter['xMax']:\n",
    "    distBins.append(chapter['xMax'])\n",
    "\n",
    "axes = dfData.DISTANCE.plot.hist(bins=distBins, #xmin=chapter['xMin'], xmax=chapter['xMax'],\n",
    "                                 figsize=(12, 6), fill=None, edgecolor='blue', rwidth=1.0, linewidth=1, zorder=10)\n",
    "axes.set_xlim((0, dfData.DISTANCE.max()))\n",
    "axes.set_xlabel('Distance')\n",
    "axes.grid(True, which='major', zorder=0)\n",
    "\n",
    "axes.grid(True, which='minor', zorder=0)\n",
    "aMTicks = axes.get_xticks()\n",
    "axes.tick_params(which='minor', grid_linestyle='-.', grid_alpha=0.6)\n",
    "axes.xaxis.set_minor_locator(pltt.MultipleLocator((aMTicks[1]-aMTicks[0])/5))\n",
    "axes.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histogramme à tranches fixes => 2 tailles de tranches (???)\n",
    "distBinWidth = 40\n",
    "distBins = np.linspace(start=0, stop=distBinWidth * int(chapter['xMax'] / distBinWidth),\n",
    "                       num=1 + int(chapter['xMax'] / distBinWidth)).tolist()\n",
    "if distBins[-1] < chapter['xMax']:\n",
    "    distBins.append(chapter['xMax'])\n",
    "\n",
    "axes = dfData.DISTANCE.plot.hist(bins=distBins, #xmin=chapter['xMin'], xmax=chapter['xMax'],\n",
    "                                 figsize=(12, 6), fill=None, edgecolor='red', rwidth=1.00, linewidth=1, zorder=10)\n",
    "\n",
    "distBinWidth = 20\n",
    "distBins = np.linspace(start=0, stop=distBinWidth * int(chapter['xMax'] / distBinWidth),\n",
    "                       num=1 + int(chapter['xMax'] / distBinWidth)).tolist()\n",
    "if distBins[-1] < chapter['xMax']:\n",
    "    distBins.append(chapter['xMax'])\n",
    "\n",
    "_ = dfData.DISTANCE.plot.hist(ax=axes, bins=distBins, #xmin=chapter['xMin'], xmax=chapter['xMax'],\n",
    "                                 figsize=(12, 6), fill=None, edgecolor='green', rwidth=0.85, linewidth=1, zorder=20)\n",
    "\n",
    "distBinWidth = 10\n",
    "distBins = np.linspace(start=0, stop=distBinWidth * int(chapter['xMax'] / distBinWidth),\n",
    "                       num=1 + int(chapter['xMax'] / distBinWidth)).tolist()\n",
    "if distBins[-1] < chapter['xMax']:\n",
    "    distBins.append(chapter['xMax'])\n",
    "\n",
    "_ = dfData.DISTANCE.plot.hist(ax=axes, bins=distBins, #xmin=chapter['xMin'], xmax=chapter['xMax'],\n",
    "                                 figsize=(12, 6), fill=None, edgecolor='blue', rwidth=0.70, linewidth=1, zorder=30)\n",
    "\n",
    "axes.set_xlim((0, dfData.DISTANCE.max()))\n",
    "axes.set_xlabel('Distance')\n",
    "axes.grid(True, which='major', zorder=0)\n",
    "\n",
    "axes.grid(True, which='minor', zorder=0)\n",
    "aMTicks = axes.get_xticks()\n",
    "axes.tick_params(which='minor', grid_linestyle='-.', grid_alpha=0.6)\n",
    "axes.xaxis.set_minor_locator(pltt.MultipleLocator((aMTicks[1]-aMTicks[0])/5))\n",
    "axes.legend([f'{deltaDist} m' for deltaDist in [40, 20, 10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histogramme à tranches fixes (KO : pas trouvé moyen forcer les ticks)\n",
    "distBinWidth = 10\n",
    "distBins = np.linspace(start=0, stop=distBinWidth * int(chapter['xMax'] / distBinWidth),\n",
    "                       num=1 + int(chapter['xMax'] / distBinWidth)).tolist()\n",
    "if distBins[-1] < chapter['xMax']:\n",
    "    distBins.append(chapter['xMax'])\n",
    "distHist, distBins = np.histogram(dfData.DISTANCE, bins=distBins)\n",
    "sDistHist = pd.Series(data=distHist, index=distBins[:-1]+distBinWidth/2)\n",
    "\n",
    "axes = sDistHist.plot.bar(figsize=(16, 4), fill=None, edgecolor='blue', width=0.9, zorder=10)\n",
    "\n",
    "#majTicksDelta = 100\n",
    "#aMajTicks = np.linspace(start=0, stop=majTicksDelta * int(chapter['xMax'] / majTicksDelta),\n",
    "#                      num=1 + int(chapter['xMax'] / majTicksDelta)).tolist()\n",
    "#axes.set_xticks(minor=False, ticks=aMajTicks)\n",
    "axes.grid(True, which='major', zorder=0)\n",
    "\n",
    "#axes.xaxis.set_minor_locator(pltt.MultipleLocator(majTicksDelta/5))\n",
    "#axes.tick_params(which='minor', grid_linestyle='-.', grid_alpha=0.6)\n",
    "#axes.grid(True, which='minor', zorder=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distHist, distBins, aMajTicks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histogramme à tranches \"optimisées\" (max(sturges, fd))\n",
    "distHist, distBins = np.histogram(dfData.DISTANCE, bins='auto', range=(0, 500))\n",
    "sDistHist = pd.Series(data=distHist, index=distBins[:-1])\n",
    "\n",
    "axes = sDistHist.plot.bar(figsize=(16, 4), fill=None, edgecolor='blue', width=0.9, zorder=10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decode stats (actual results) from MCDS work folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#_ = implib.reload(ads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results set to store results into.\n",
    "miCustCols = pd.MultiIndex.from_tuples([('id', 'ExecCase', 'Value')])\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols, data=dict(en=['ExecCase'], fr=['CasExec']))\n",
    "\n",
    "results = ads.MCDSAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans, sampleIndCol='SampNum',\n",
    "                                     distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                     surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis engine\n",
    "mcds = ads.MCDSEngine(workDir='refout/dist-order-sens-min',\n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process folders in engine work folder.\n",
    "for folder in os.listdir(mcds.workDir):\n",
    "    \n",
    "    # Skip folders that are not MCDS run ones.\n",
    "    folderPath = os.path.join(mcds.workDir, folder)\n",
    "    if not os.path.isdir(folderPath):\n",
    "        continue\n",
    "    if os.path.splitext(folder)[1] or 'stats.txt' not in os.listdir(folderPath):\n",
    "        print(f'Skipping {folderPath}, not an MCDS.exe run folder with a stats.txt file')\n",
    "        continue\n",
    "        \n",
    "    # Tell the engine were it has run (even it does not rember it ;-)\n",
    "    #_ = mcds.setupRunFolder(forceSubFolder=folder)\n",
    "    \n",
    "    # Decode results.\n",
    "    sRes = mcds.decodeStats(folder)\n",
    "    print()\n",
    "    \n",
    "    # Store them for later.\n",
    "    sHead = pd.Series(data=[folder], index=miCustCols)\n",
    "    results.append(sRes, sCustomHead=sHead)\n",
    "\n",
    "# Tadaaaaaaa !\n",
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfTransData('en').to_excel(pl.Path('tmp', 'dist-order-sens-auto-results.xlsx'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Development of AnalysisResultsSet.compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfl = pd.DataFrame([dict(i=1, a=1.0, b=2.0, c=np.nan),\n",
    "                    dict(i=2, a=2.0, b=3.0, c=4.0),\n",
    "                    dict(i=3, a=4.0, b=5.0, c=np.nan),\n",
    "                    dict(i=4, a=np.nan, b=6.0, c=-7.5)])\n",
    "dfl.set_index('i', inplace=True)\n",
    "dfr = pd.DataFrame([dict(i=0, a=0.0, b=np.nan, c=2.0),\n",
    "                    dict(i=2, a=2.01, b=3.0, c=np.nan),\n",
    "                    dict(i=3, a=np.nan, b=5.01, c=np.nan),\n",
    "                    dict(i=4, a=np.nan, b=6.0, c=-7.5)])\n",
    "dfr.set_index('i', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfl1 = dfl.join(dfr[['a']], rsuffix='_r', how='outer').drop(columns='a_r')\n",
    "dfl1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfr1 = dfr.join(dfl[['a']], rsuffix='_l', how='outer').drop(columns='a_l')\n",
    "dfr1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nColLevels = dfl1.columns.nlevels\n",
    "KTmpCol = 'tmp' if nColLevels == 1 else tuple('tmp{}'.format(i) for i in range(nColLevels))\n",
    "dfd = dfl1.copy()\n",
    "for col2Diff in dfl1.columns:\n",
    "    dfd[KTmpCol] = dfr1[col2Diff]\n",
    "    dfd[col2Diff] = dfd[[col2Diff, KTmpCol]].apply(closeness, axis='columns')\n",
    "    dfd.drop(columns=[KTmpCol], inplace=True)\n",
    "dfd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfd.loc[dfl[~dfl.index.isin(dfr.index)].index, :] = 0\n",
    "dfd.loc[dfr[~dfr.index.isin(dfl.index)].index, :] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropNans = True\n",
    "#sbRows2Drop = (dfd > 2 | dfd.isnull()).all(axis='columns')\n",
    "sbRows2Drop = dfd.apply(lambda s: s > 2 | s.isnull(), axis='index').all(axis='columns')\n",
    "#sbRows2Drop = dfd.applymap(lambda v: v > 2 or pd.isnull(v)).all(axis='columns')\n",
    "sbRows2Drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfd.drop(dfd[sbRows2Drop].index, axis='index', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfr1 = dfr.append(dfl[~dfl.index.isin(dfr.index)], sort=False)\n",
    "dfr1.sort_index(inplace=True)\n",
    "dfr1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate stats columns translation file\n",
    "\n",
    "(from documentation stats & modules specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tgtTransFileName = 'tmp/stat-mod-trans.auto.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Translator(object):\n",
    "    \n",
    "    def __init__(self, dTrans, lang='en'):\n",
    "        assert 'en' in dTrans, 'At least \"en\" translation must be defined'\n",
    "        self.dTrans = dTrans\n",
    "        self.setLang(lang)\n",
    "        \n",
    "    def setLang(self, lang):\n",
    "        self.lang = lang.lower()\n",
    "        assert self.lang in ['en', 'fr'], 'No support for \"{}\" language'.format(lang)\n",
    "        \n",
    "    def __call__(self, s):\n",
    "        return self.dTrans.get(self.lang, self.dTrans['en']).get(s, self.dTrans['en'].get(s, s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DFigureTrans = \\\n",
    "    dict(en=dict(Value='', Cv='CoefVar', Lcl='Min', Ucl='Max', Df='DoF'),\n",
    "         fr=dict(Value='', Cv='CoefVar', Lcl='Min', Ucl='Max', Df='DegLib'))\n",
    "\n",
    "figtr = Translator(DFigureTrans, lang='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DStatisticTrans = \\\n",
    "    dict(en={ 'number of observations (n)': 'NObs',\n",
    "              'number of samples (k)': 'NSamp',\n",
    "              'effort (L or K or T)': 'Effort',\n",
    "              'encounter rate (n/L or n/K or n/T)': 'EncRate',\n",
    "              'left truncation distance': 'LeftTruncDist',\n",
    "              'right truncation distance (w)': 'RightTruncDist',\n",
    "              'total number of parameters (m)': 'TotNumPars',\n",
    "              'AIC value': 'AIC',\n",
    "              'chi-square test probability (distance set 1)': 'Chi2 P 1',\n",
    "              'chi-square test probability (distance set 2)': 'Chi2 P 2',\n",
    "              'chi-square test probability (distance set 3)': 'Chi2 P 3',\n",
    "              'f(0) or h(0)': 'f/h(0)',\n",
    "              'probability of detection (Pw)': 'PDetec',\n",
    "              'effective strip width (ESW) or effective detection radius (EDR)': 'EDR/ESW',\n",
    "              'AICc': 'AICc',\n",
    "              'BIC': 'BIC',\n",
    "              'Log likelihood': 'LogLhood',\n",
    "              'Kolmogorov-Smirnov test probability': 'KS P',\n",
    "              'Cramér-von Mises (uniform weighting) test probability': 'CvM Uw P',\n",
    "              'Cramér-von Mises (cosine weighting) test probability': 'CvM Cw P',\n",
    "              'key function type': 'KeyFn',\n",
    "              'adjustment series type': 'AdjSer',\n",
    "              'number of key function parameters (NKP)': 'NumKFnPars',\n",
    "              'number of adjustment term parameters (NAP)': 'NumASerPars',\n",
    "              'number of covariate parameters (NCP)': 'NumCovars',\n",
    "              'estimated value of A(1) adjustment term parameter': 'EstA(1)',\n",
    "              'estimated value of A(2) adjustment term parameter': 'EstA(2)',\n",
    "              'estimated value of A(3) adjustment term parameter': 'EstA(3)',\n",
    "              'estimated value of A(4) adjustment term parameter': 'EstA(4)',\n",
    "              'estimated value of A(5) adjustment term parameter': 'EstA(5)',\n",
    "              'estimated value of A(6) adjustment term parameter': 'EstA(6)',\n",
    "              'estimated value of A(7) adjustment term parameter': 'EstA(7)',\n",
    "              'estimated value of A(8) adjustment term parameter': 'EstA(8)',\n",
    "              'estimated value of A(9) adjustment term parameter': 'EstA(9)',\n",
    "              'estimated value of A(10) adjustment term parameter': 'EstA(10)',\n",
    "              'average cluster size': 'AvgClustSz',\n",
    "              'size-bias regression correlation (r)': 'SzBias RegCorr',\n",
    "              'p-value for correlation significance (r-p)': 'CorSignPVal',\n",
    "              'estimate of expected cluster size corrected for size bias': 'EstExpFixedCluSz',\n",
    "              'density of clusters (or animal density if non-clustered)': 'DensClu',\n",
    "              'density of animals': 'Density',\n",
    "              'number of animals, if survey area is specified': 'Number',\n",
    "              'bootstrap density of clusters': 'BootsDensClu',\n",
    "              'bootstrap density of animals': 'BootDensity',\n",
    "              'bootstrap number of animals': 'BootNumber' },\n",
    "         fr={ 'number of samples (k)': 'NEchant',\n",
    "              'encounter rate (n/L or n/K or n/T)': 'TxContact',\n",
    "              'left truncation distance': 'DistTroncGche',\n",
    "              'right truncation distance (w)': 'DistTroncDte',\n",
    "              'total number of parameters (m)': 'NbTotPars',\n",
    "              'Log likelihood': 'LogProba',\n",
    "              'key function type': 'FnClé',\n",
    "              'adjustment series type': 'SérAjust',\n",
    "              'number of key function parameters (NKP)': 'NbParsFnClé',\n",
    "              'number of adjustment term parameters (NAP)': 'NbParsSérAjust',\n",
    "              'number of covariate parameters (NCP)': 'NbCovars',\n",
    "              'average cluster size': 'TailMoyClust',\n",
    "              'size-bias regression correlation (r)': 'CorrReg BiaisTail',\n",
    "              'p-value for correlation significance (r-p)': 'PVal SignifCorr',\n",
    "              'estimate of expected cluster size corrected for size bias': 'TailCorrCluAttEst',\n",
    "              'density of animals': 'Densité',\n",
    "              'number of animals, if survey area is specified': 'Nombre',\n",
    "              'bootstrap density of clusters': 'BootsDensClu',\n",
    "              'bootstrap density of animals': 'DensitéBoot',\n",
    "              'bootstrap number of animals': 'NombreBoot' })\n",
    "\n",
    "statr = Translator(DStatisticTrans, lang='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatModTrans = ads.MCDSEngine.statModCols().to_frame()\n",
    "dfStatModTrans.reset_index(drop=True, inplace=True)\n",
    "dfStatModTrans.rename(columns={ 0: 'Module', 1: 'Statistic', 2: 'Figure' }, inplace=True)\n",
    "for lang in ['en', 'fr']:\n",
    "    figtr.setLang(lang)\n",
    "    statr.setLang(lang)\n",
    "    dfStatModTrans[lang] = \\\n",
    "        dfStatModTrans.apply(lambda sRow: '{} {}'.format(figtr(sRow.Figure), statr(sRow.Statistic)).strip(),\n",
    "                             axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatModTrans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatModTrans.to_csv(tgtTransFileName, sep='\\t', index=False)\n",
    "tgtTransFileName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(index=ads.MCDSAnalysis.MIRunColumns,\n",
    "             data=dict(en=['ModKeyFn', 'ModAdjSer', 'ModChcCrit', 'ConfInterv', 'LeftTrunc', 'RightTrunc',\n",
    "                           'FitDistCuts', 'DiscrDistCuts', 'RunCode', 'StartTime', 'ElapsedTime', 'RunFolder'],\n",
    "                       fr=['FnCléMod', 'SérAjustMod', 'CritChxMod', 'IntervConf', 'TroncGauche', 'TroncDroite',\n",
    "                           'TranchDistFit', 'TranchDistDiscr', 'CodeExec', 'DébutExec', 'DuréeExec', 'DossierExec']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatModTransExt = pd.read_csv(tgtTransFileName, sep='\\t')\n",
    "dfStatModTransExt.set_index(['Module', 'Statistic', 'Figure'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang = 'fr'\n",
    "dTrans = dfStatModTransExt.set_index(['Module', 'Statistic', 'Figure'])[lang].to_dict()\n",
    "results.dfData.columns = [dTrans.get(col, col) for col in results.dfData.columns]\n",
    "results.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatModTransExt.set_index(['Module', 'Statistic', 'Figure'])[lang].to_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test case class\n",
    "\n",
    "(no use actually : pd.DataFrame already does the job !)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Super-class for test cases\n",
    "class TestCase(object):\n",
    "    def __init__(self, **attrs):\n",
    "        if not hasattr(self.__class__, 'AttributeNames'):\n",
    "            self.__class__.AttributeNames = set(attrs.keys())\n",
    "        else:\n",
    "            assert set(attrs.keys()) == self.AttributeNames, \\\n",
    "                   'Some attribute name not in frozen set {{{}}}'.format(','.join(self.AttributeNames))\n",
    "        for attrName, AttrValue in attrs.items():\n",
    "            setattr(self, attrName, AttrValue)\n",
    "    def __repr__(self):\n",
    "        return '{}({})'.format(self.__class__.__name__, ','.join('{}:{}'.format(k, v) for k, v in self.__dict__.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test this super-class.\n",
    "class TCTest(TestCase):\n",
    "    pass\n",
    "\n",
    "tstTestCases = list()\n",
    "tstTestCases.append(TCTest(x=1, y='a')) # Define attributes\n",
    "tstTestCases.append(TCTest(x=2, y='b')) # Check attributes\n",
    "try:\n",
    "    tstTestCases.append(TCTest(x=2, z=None)) # Refuse new attributes\n",
    "    assert False, 'Error: New attributes should be refused'\n",
    "except AssertionError as exc:\n",
    "    print('Good refuse of new attributes:', exc)\n",
    "    \n",
    "[str(tc) for tc in tstTestCases]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MCDS output decoding : stats file\n",
    "\n",
    "TODO: Add french translation of variables / parameters names and descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Name and description of stats table columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileName = 'mcds/stat-row-specs.txt'\n",
    "\n",
    "fStatRowSpecs = open(fileName, mode='r', encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "statRowSpecLines = [line.rstrip('\\n') for line in fStatRowSpecs.readlines() if not line.startswith('#')]\n",
    "statRowSpecs =  [(statRowSpecLines[i].strip(), statRowSpecLines[i+1].strip()) \\\n",
    "                 for i in range(0, len(statRowSpecLines)-2, 3)]\n",
    "dfStatRowSpecs = pd.DataFrame(columns=['Name', 'Description'], data=statRowSpecs).set_index('Name')\n",
    "\n",
    "dfStatRowSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatRowSpecs.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Number and description of the modules and relevant stats\n",
    "\n",
    "(Module and Statistic columns of the table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileName = 'mcds/stat-mod-specs.txt'\n",
    "\n",
    "fStatModSpecs = open(fileName, mode='r', encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nMaxAdjParams = 10\n",
    "\n",
    "statModSpecLines = [line.rstrip('\\n') for line in fStatModSpecs.readlines() if not line.startswith('#')]\n",
    "reModSpecNumName = re.compile('(.+) – (.+)')\n",
    "statModSpecs = list()\n",
    "moModule = None\n",
    "for line in statModSpecLines:\n",
    "    if not line:\n",
    "        continue\n",
    "    if moModule is None:\n",
    "        moModule = reModSpecNumName.match(line.strip())\n",
    "        continue\n",
    "    if line == ' ':\n",
    "        moModule = None\n",
    "        continue\n",
    "    moStatistic = reModSpecNumName.match(line.strip())\n",
    "    modNum, modDesc, statNum, statDescNotes = \\\n",
    "        moModule.group(1), moModule.group(2), moStatistic.group(1), moStatistic.group(2)\n",
    "    for i in range(len(statDescNotes)-1, -1, -1):\n",
    "        if not re.match('[\\d ,]', statDescNotes[i]):\n",
    "            statDesc = statDescNotes[:i+1]\n",
    "            statNotes = statDescNotes[i+1:].replace(' ', '')\n",
    "            break\n",
    "    modNum = int(modNum)\n",
    "    if statNum.startswith('101 '):\n",
    "        for num in range(nMaxAdjParams): # Assume no more than that ... a bit hacky !\n",
    "            statModSpecs.append((modNum, modDesc, 101+num, # Make statDesc unique for later indexing\n",
    "                                 statDesc.replace('each', 'A({})'.format(num+1)), statNotes))\n",
    "    else:\n",
    "        statNum = int(statNum)\n",
    "        if modNum == 2 and statNum == 3: # Actually, there are 0 or 3 of these ...\n",
    "            for num in range(3):\n",
    "                statModSpecs.append((modNum, modDesc, num+201,\n",
    "                                     # Change statNum & Make statDesc unique for later indexing\n",
    "                                     statDesc+' (distance set {})'.format(num+1), statNotes))\n",
    "        else:\n",
    "            statModSpecs.append((modNum, modDesc, statNum, statDesc, statNotes))\n",
    "dfStatModSpecs = pd.DataFrame(columns=['modNum', 'modDesc', 'statNum', 'statDesc', 'statNotes'],\n",
    "                              data=statModSpecs).set_index(['modNum', 'statNum'])\n",
    "\n",
    "dfStatModSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modules\n",
    "dfStatModSpecs.modDesc.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Notes on module statistics\n",
    "\n",
    "(more infos explainig how to use or not the 5 last columns: Value, Cv, Lcl, Ucl, Df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileName = 'mcds/stat-mod-notes.txt'\n",
    "\n",
    "fStatModNotes = open(fileName, mode='r', encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "statModNoteLines = [line.rstrip('\\n') for line in fStatModNotes.readlines() if not line.startswith('#')]\n",
    "statModNotes =  [(int(line[:2]), line[2:].strip()) for line in statModNoteLines if line]\n",
    "\n",
    "dfStatModNotes = pd.DataFrame(data=statModNotes, columns=['Note', 'Text']).set_index('Note')\n",
    "\n",
    "dfStatModNotes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Read table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng = mcds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng.statsFileName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatRows = pd.read_csv(eng.statsFileName, sep=' +', engine='python', names=dfStatRowSpecs.index)\n",
    "dfStatRows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Decode table\n",
    "\n",
    "Warning: We assume there's only 1 '0' stratum, only 1 '0' sample and only 1 '1' estimator '1'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### a. Remove Stratum, Sample and Estimator columns\n",
    "\n",
    "(see warning above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStatRows.drop(columns=['Stratum', 'Sample', 'Estimator'], inplace=True)\n",
    "dfStatRows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### b. Cleanup N/A data\n",
    "\n",
    "(according to 'notes' on stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Empilage des \"chiffres\" (Figures) Value, Cv, Lcl, Ucl, Df pour chaque statistique / module\n",
    "dfStats = dfStatRows.set_index(['Module', 'Statistic'], append=True).stack() \\\n",
    "                    .reset_index().rename(columns={'level_0': 'id', 'level_3': 'Figure', 0: 'Value'})\n",
    "dfStats.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. Fix multiple Module=2 & Statistic=3 rows (before joining with self.DfStatModSpecs)\n",
    "newStatNum = 200\n",
    "for lbl, sRow in dfStats[(dfStats.Module == 2) & (dfStats.Statistic == 3)].iterrows():\n",
    "    if dfStats.loc[lbl, 'Figure'] == 'Value':\n",
    "        newStatNum += 1\n",
    "    dfStats.loc[lbl, 'Statistic'] = newStatNum\n",
    "dfStats[(dfStats.Module == 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ajout des colonnes de description/nommage des modules et statistiques\n",
    "dfStats = dfStats.join(dfStatModSpecs, on=['Module', 'Statistic'])\n",
    "dfStats.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfStats[(dfStats.Module == 2) & (dfStats.Statistic > 200)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vérification que les chiffres sans objet le sont vraiment (tous à 0.0 ?)\n",
    "# Attention: Il doit y avoir un bug dans MCDS avec Module 2 / Statistic 10x : certains Cv ne sont pas nuls ...\n",
    "sKeepOnlyValueFig = ~dfStats.statNotes.str.contains('1')\n",
    "sFigs2Drop = (dfStats.Figure != 'Value') & sKeepOnlyValueFig\n",
    "assert ~dfStats[sFigs2Drop & ((dfStats.Module != 2) | (dfStats.Statistic < 100))].Value.any(), \\\n",
    "       'Attention: Des chiffres supposés \"sans objet\" on des valeurs non nulles !'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2nde vérif. visuelle\n",
    "dfStats[sFigs2Drop & dfStats.Value != 0].sort_values(by='Value', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Suppression des lignes / chiffres sans objet.\n",
    "dfStats.drop(dfStats[sFigs2Drop].index, inplace=True)\n",
    "dfStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStats.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStats = dfStats.reindex(columns=['modDesc', 'statDesc', 'Figure', 'Value'])\n",
    "dfStats.set_index(['modDesc', 'statDesc', 'Figure'], inplace=True)\n",
    "dfStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStats.T.iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List MCDS warnings\n",
    "\n",
    "(from massive analysis runs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# $ cd donnees/cretes-cantal/201006-1527  # Un dossier de travail pour analyses auto\n",
    "# $ find . -name \"log.txt\" | xargs grep \"Warning\n",
    "# Ex.\n",
    "# ./TurdMeru-m-haz-cos-l50-r200-olv9yrf2/log.txt:      ** Warning: Parameter  2 is at an upper bound. **\n",
    "# ./TurdMeru-m-haz-cos-l50-r250-4v1smzl8/log.txt:      ** Warning: Parameters are being constrained to obtain monotonicity. **\n",
    "# ./TurdMeru-m-haz-cos-la-0v5ylg74/log.txt:      ** Warning: Parameters are being constrained to obtain monotonicity. **\n",
    "# ./TrogTrog-m-uni-pol-ra-ma-pgspzdn1/log.txt:** Warning: convergence failure **\n",
    "\n",
    "with open('donnees/cretes-cantal/mcds-warnings.log', 'r') as file:\n",
    "    lines = file.readlines()\n",
    "    \n",
    "dfw = pd.DataFrame(data=[line.split(':') for line in lines], columns=['fpn', 'z', 'x', 'y'])\n",
    "dfw[['t', 'fn']] = dfw.fpn.apply(lambda s: pd.Series(s[2:].split('/')[:2]))\n",
    "dfw[['s', 'a']] = dfw.fn.apply(lambda s: pd.Series([s[:10], s[11:]]))\n",
    "dfw['w'] = dfw[['x', 'y']].apply(lambda s: s.x + (' ' + s.y.strip() if s.y else ''), axis='columns').apply(str.strip)\n",
    "dfw.w = dfw.w.apply(lambda s: s.replace(' **', ''))\n",
    "dfw.drop(columns=['fn', 'z', 'x', 'y'], inplace=True)\n",
    "#dfw.set_index(['t', 's', 'a', 'fpn'], inplace=True)\n",
    "\n",
    "dfw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict(warning_analyses=dfw.fpn.nunique(), warning_types=dfw.w.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dfw[['a', 'w']].groupby('a').count().value_counts().reset_index().T.to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# $ cd donnees/cretes-cantal/201006-1527  # Un dossier de travail pour analyses auto\n",
    "# $ find .-name \"*log.txt\" >../mcds-runs.log\n",
    "# Ex:\n",
    "# ./AlauArve-m-haz-cos-bqia9v69/log.txt\n",
    "# ./AlauArve-m-haz-cos-l30-r100-i6y4g1l8/log.txt\n",
    "# ./AlauArve-m-haz-cos-l30-r150-nbntf7tm/log.txt\n",
    "# ./AlauArve-m-haz-cos-l30-r200-wlrrb2f5/log.txt\n",
    "# ./AlauArve-m-haz-cos-l30-r250-tbd2sgrd/log.txt\n",
    "# ./AlauArve-m-haz-cos-l50-r100-uzc695ac/log.txt\n",
    "# ./AlauArve-m-haz-cos-l50-r150-2jiy8n9b/log.txt\n",
    "\n",
    "\n",
    "with open('donnees/cretes-cantal/mcds-runs.log', 'r') as file:\n",
    "    lines = file.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfr = pd.DataFrame(index=[line.strip() for line in lines])\n",
    "dfr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict(analyses=len(dfr), warning_analyses=dfw.fpn.nunique(), pct_warning_analyses=100 * dfw.fpn.nunique() / len(dfr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfw = dfr.join(dfw.set_index('fpn'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfw.w = dfw.w.apply(lambda s: s.replace('Parameter  1', 'Parameter  n').replace('Parameter  2', 'Parameter  n'))\n",
    "dfw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfp = dfw[['t', 's', 'a', 'w']].groupby(['t', 's', 'a']).apply(lambda dfg: ' # '.join(dfg.w.sort_values()))\n",
    "dfp = dfp.reset_index().rename(columns={0: 'w'})\n",
    "dfp['n'] = dfp.w.apply(lambda s: s.count('#')+1)\n",
    "dfp.sort_values(by=['n', 'w'], ascending=True, inplace=True)\n",
    "dfp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfp.w.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfp.w.value_counts().reset_index().to_excel('tmp/__.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfp.to_excel('tmp/_.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pd.read_excel('tmp/__.xlsx', sheet_name='top24').to_markdown(index=False).replace('   ', ''))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distance cut specs for MCDS\n",
    "\n",
    "* Mise au point\n",
    "* tests unitaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distanceCutSpecs(minDist, maxDist, distCuts):\n",
    "    \n",
    "    distCutSpecs = ''\n",
    "        \n",
    "    #distCuts = params['distCuts']\n",
    "    if distCuts is not None:\n",
    "\n",
    "        if isinstance(distCuts, list) and minDist is not None and maxDist is not None:\n",
    "            distCutSpecs += ' /Int=' + ','.join(str(d) for d in [minDist] + distCuts + [maxDist])\n",
    "        elif isinstance(distCuts, int):\n",
    "            distCutSpecs += ' /NClass=' + str(distCuts)\n",
    "\n",
    "    if not isinstance(distCuts, list): # None or int\n",
    "\n",
    "        #minDist = params['minDist']\n",
    "        if minDist is not None:\n",
    "            distCutSpecs += ' /Left=' + str(minDist)\n",
    "\n",
    "        #maxDist = params['maxDist']\n",
    "        if maxDist is not None:\n",
    "            distCutSpecs += ' /Width=' + str(maxDist)\n",
    "            \n",
    "    return distCutSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert distanceCutSpecs(minDist=None, maxDist=None, distCuts=None) == ''\n",
    "\n",
    "assert distanceCutSpecs(minDist=5, maxDist=None, distCuts=None) == ' /Left=5'\n",
    "assert distanceCutSpecs(minDist=None, maxDist=100, distCuts=None) == ' /Width=100'\n",
    "assert distanceCutSpecs(minDist=25.2, maxDist=100.5, distCuts=None) == ' /Left=25.2 /Width=100.5'\n",
    "\n",
    "assert distanceCutSpecs(minDist=None, maxDist=None, distCuts=3) == ' /NClass=3'\n",
    "assert distanceCutSpecs(minDist=None, maxDist=300, distCuts=8) == ' /NClass=8 /Width=300'\n",
    "assert distanceCutSpecs(minDist=20, maxDist=None, distCuts=8) == ' /NClass=8 /Left=20'\n",
    "assert distanceCutSpecs(minDist=20, maxDist=300, distCuts=8) == ' /NClass=8 /Left=20 /Width=300'\n",
    "\n",
    "assert distanceCutSpecs(minDist=20, maxDist=300, distCuts=[100, 200, 230, 290]) == ' /Int=20,100,200,230,290,300'\n",
    "assert distanceCutSpecs(minDist=None, maxDist=None, distCuts=[1, 2, 3]) == '' # min & maxDist have to be both defined\n",
    "assert distanceCutSpecs(minDist=0, maxDist=None, distCuts=[1, 2, 3]) == '' # min & maxDist have to be both defined\n",
    "assert distanceCutSpecs(minDist=None, maxDist=4, distCuts=[1, 2, 3]) == '' # min & maxDist have to be both defined"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data tools development"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### addAbsenceSightings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfInSightings = dfObsIndiv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transectCol = 'Point'\n",
    "taxonCol = 'Espece'\n",
    "sampleCols = ['Passage', 'Adulte', 'Duree']\n",
    "\n",
    "# The set of expected taxa ... of which we'll look for abscence on every location\n",
    "expectedTaxa = list(dfObsIndiv[taxonCol].unique())\n",
    "', '.join(expectedTaxa), len(expectedTaxa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add \"abscence\" sightings to field data collected on transects for a given sample\n",
    "#def addAbsenceSightings(dfInSightings, transectCol, taxonCol, expectedTaxa, sampleCols):\n",
    "    \n",
    "def absenceSightings(taxonCol, taxon, dAbscSightTmpl):\n",
    "    dAbscSight = dAbscSightTmpl.copy()\n",
    "    dAbscSight[taxonCol] = taxon\n",
    "    return dAbscSight\n",
    "\n",
    "assert not dfInSightings.empty, 'Error : Empty sightings data to complete !'\n",
    "\n",
    "ldfAbscSightings = list()\n",
    "\n",
    "# Use 1st sightings of the sample to build the absence sightings prototype\n",
    "# (all null columns except for the sample identification ones)\n",
    "dAbscSightTmpl = dfInSightings.iloc[0].to_dict()\n",
    "dAbscSightTmpl.update({ k: None for k in dAbscSightTmpl.keys() if k not in sampleCols })\n",
    "\n",
    "# For each transect\n",
    "for transect in dfInSightings[transectCol].unique():\n",
    "\n",
    "    # Update absence sightings template with transect id\n",
    "    dAbscSightTmpl.update({ transectCol: transect })\n",
    "    \n",
    "    # Generate the absence sightings from it : 1 per lacking taxon\n",
    "    lackingTaxa = set(expectedTaxa) - set(dfInSightings.loc[dfInSightings[transectCol] == transect, taxonCol].unique())\n",
    "    dfAbscSights = pd.DataFrame([absenceSightings(taxonCol, txn, dAbscSightTmpl) for txn in lackingTaxa])\n",
    "    \n",
    "    ldfAbscSightings.append(dfAbscSights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concat all data frames into one.\n",
    "dfOutSightings = pd.concat([dfInSightings] + ldfAbscSightings)\n",
    "\n",
    "# Reset index (for unique labels).\n",
    "dfOutSightings.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfOutSightings), len(dfInSightings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distance troncations : auto-generation of variants\n",
    "\n",
    "(at least a try ...)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data for playing : Individualised ones ...\n",
    "\n",
    "(dfObsIndiv from somewhere in valtests.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv.groupby('Espece').size().sort_values(ascending=False)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndSpc = dfObsIndiv[dfObsIndiv.Espece == 'Merle noir'].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histogramme uniforme\n",
    "_ = dfObsIndSpc.distMem.hist(figsize=(16, 3), bins=40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Empirical distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.distributions.empirical_distribution as sted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ecdf = sted.ECDF(dfObsIndSpc.distMem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sEcdf = pd.Series({ x : ecdf(x) for x in dfObsIndSpc.distMem.unique() }).sort_index()\n",
    "_ = sEcdf.plot(figsize=(16, 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quantiles : 2.5, 5 et 10%, left and right sides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aqLims = np.array([0.025, 0.05, 0.1, 0.9, 0.95, 0.975])\n",
    "aqLims * len(dfObsIndSpc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.quantile(a=dfObsIndSpc.distMem, q=aqLims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndSpc[dfObsIndSpc.distMem <= 11.61]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Brute force combination algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lParams = list() # of dict(ltr=<left trunc dist or None>, rtr=<right trunc dist or None>, nc=<nb of cuts>)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aqtlLims = np.array([1.25, 2.5, 3.75, 5, 7.5, 10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sLeftTruncs = pd.Series(index=aqtlLims, data=np.percentile(a=dfObsIndSpc.distMem, q=aqtlLims))\n",
    "for leftPct, leftTrunc in sLeftTruncs.items():\n",
    "    nRetSights = len(dfObsIndSpc[dfObsIndSpc.distMem >= leftTrunc])\n",
    "    sqrNRetSights = math.sqrt(nRetSights)\n",
    "    for nCuts in [2*sqrNRetSights/3, sqrNRetSights, 3*sqrNRetSights/2]:\n",
    "        lParams.append(dict(ltr=leftTrunc, rtr=None, nc=round(nCuts), nrs=nRetSights, pct=100-leftPct))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sRightTruncs = pd.Series(index=100-aqtlLims, data=np.percentile(a=dfObsIndSpc.distMem, q=100-aqtlLims)).sort_index()\n",
    "for rightPct, rightTrunc in sRightTruncs.items():\n",
    "    nRetSights = len(dfObsIndSpc[dfObsIndSpc.distMem <= rightTrunc])\n",
    "    sqrNRetSights = math.sqrt(nRetSights)\n",
    "    for nCuts in [2*sqrNRetSights/3, sqrNRetSights, 3*sqrNRetSights/2]:\n",
    "        lParams.append(dict(ltr=None, rtr=rightTrunc, nc=round(nCuts), nrs=nRetSights, pct=rightPct))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ... etc ... but, why not use an optimisation engine then ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(lParams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimising algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndSpc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcds = ads.MCDSEngine(workDir=os.path.join('ACDC', '2019-nat-opt'),\n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleDecFields = ['Effort', 'Distance']\n",
    "\n",
    "dAreaInfo = dict(Zone='ACDC', Surface=2400) # ha\n",
    "dfObsIndSpc = ads.addSurveyAreaInfo(dfObsIndSpc, dAreaInfo=dAreaInfo)\n",
    "\n",
    "dfObsIndSpc.rename(columns=dict(distMem='Distance'), inplace=True)\n",
    "dfObsIndSpc.sort_values(by='Point', inplace=True)\n",
    "\n",
    "sampDataSet = ads.DataSet(dfObsIndSpc, decimalFields=sampleDecFields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sSamp = dfObsIndSpc.iloc[0]\n",
    "abrvSpe = ''.join(word[:4].title() for word in sSamp['Espece'].split(' '))\n",
    "sampAbbrev = '{}-{}-{}-{}'.format(abrvSpe, sSamp.Passage.replace('+', ''),\n",
    "                                  sSamp.Adulte.replace('+', ''), sSamp['Duree'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KPreEstimCrit = 'AICC'\n",
    "KPreCVInterval = 95\n",
    "\n",
    "def dsAnalyser3(aParams, mcdsEngine, sampDataSet, sampAbbrev, keyFn, adjSer):\n",
    "\n",
    "    minDist = aParams[0]\n",
    "    maxDist = aParams[1]\n",
    "    fitDistCuts = round(aParams[2])\n",
    "    print(minDist, maxDist, fitDistCuts)\n",
    "    \n",
    "    modAbbrev = keyFn[:3].lower() + '-' + adjSer[:3].lower()\n",
    "\n",
    "    analysis = ads.MCDSAnalysis(engine=mcdsEngine, sampleDataSet=sampDataSet,\n",
    "                                name=sampAbbrev + '-' + modAbbrev, logData=False,\n",
    "                                estimKeyFn=keyFn, estimAdjustFn=adjSer,\n",
    "                                estimCriterion=KPreEstimCrit, cvInterval=KPreCVInterval,\n",
    "                                minDist=minDist, maxDist=maxDist, fitDistCuts=fitDistCuts)\n",
    "    \n",
    "    sResult = analysis.submit().getResults()\n",
    "\n",
    "    aic = sResult[('detection probability', 'AIC value', 'Value')]\n",
    "    \n",
    "    return aic\n",
    "\n",
    "def dsAnalyser2(aParams, mcdsEngine, sampDataSet, sampAbbrev, keyFn, adjSer, fitDistCuts):\n",
    "\n",
    "    minDist = aParams[0]\n",
    "    maxDist = aParams[1]\n",
    "    print(minDist, maxDist)\n",
    "    \n",
    "    modAbbrev = keyFn[:3].lower() + '-' + adjSer[:3].lower()\n",
    "\n",
    "    analysis = ads.MCDSAnalysis(engine=mcdsEngine, sampleDataSet=sampDataSet,\n",
    "                                name=sampAbbrev + '-' + modAbbrev, logData=False,\n",
    "                                estimKeyFn=keyFn, estimAdjustFn=adjSer,\n",
    "                                estimCriterion=KPreEstimCrit, cvInterval=KPreCVInterval,\n",
    "                                minDist=minDist, maxDist=maxDist, fitDistCuts=fitDistCuts)\n",
    "    \n",
    "    sResult = analysis.submit().getResults()\n",
    "    #print(sResult.to_dict())\n",
    "\n",
    "    #return sResult[('detection probability', 'AIC value', 'Value')]\n",
    "    return sResult[('detection probability', 'AICc', 'Value')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "adjSer = 'COSINE'\n",
    "keyFn = 'HNORMAL'\n",
    "#keyFn = 'HAZARD'\n",
    "#keyFn = 'UNIFORM'\n",
    "#keyFn = 'NEXPON'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Juste une analyse pour tester la fonction à r (AIC)\n",
    "#              minDist, maxDist, fitDistCuts\n",
    "aParams = np.array([0, 250, round(math.sqrt(len(sampDataSet.dfData)))])\n",
    "dsAnalyser3(aParams, mcds, sampDataSet, sampAbbrev, keyFn, adjSer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Et maintenant, on lance l'optimisation.\n",
    "#              minDist, maxDist, fitDistCuts\n",
    "#maxMinDist, minMaxDist = np.percentile(a=dfObsIndSpc.Distance, q=[20, 80])\n",
    "#maxMinDist, minMaxDist = np.percentile(a=dfObsIndSpc.Distance, q=[40, 60])\n",
    "maxMinDist, minMaxDist = np.percentile(a=dfObsIndSpc.Distance, q=[49, 51])\n",
    "sqrNRetSights = math.sqrt(len(dfObsIndSpc))\n",
    "minFitDistCuts, maxFitDistCuts = round(2*sqrNSights/3), round(3*sqrNRetSights/2)\n",
    "paramBounds = [(0, maxMinDist), (minMaxDist, dfObsIndSpc.Distance.max()), (minFitDistCuts, maxFitDistCuts)]\n",
    "paramBounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import optimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dOptRes = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitDistCuts = 12\n",
    "dOptRes['shgo'] = optimize.shgo(func=dsAnalyser2, bounds=paramBounds[:2], iters=2,\n",
    "                                args=(mcds, sampDataSet, sampAbbrev, keyFn, adjSer, fitDistCuts))\n",
    "dOptRes['shgo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dOptRes['shgo'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dOptRes['shgo'] = optimize.shgo(func=dsAnalyser3, bounds=paramBounds, iters=2,\n",
    "                                args=(mcds, sampDataSet, sampAbbrev, keyFn, adjSer))\n",
    "dOptRes['shgo']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dOptRes['da'] = optimize.dual_annealing(func=dsAnalyser3, bounds=paramBounds,\n",
    "                                        args=(mcds, sampDataSet, sampAbbrev, keyFn, adjSer))\n",
    "dOptRes['da']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dOptRes['de'] = optimize.differential_evolution(func=dsAnalyser3, bounds=paramBounds,\n",
    "                                                args=(mcds, sampDataSet, sampAbbrev, keyFn, adjSer))\n",
    "dOptRes['de']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dOptRes['bh'] = optimize.basinhopping(func=dsAnalyser3, x0=[(mx+mn)/2 for mx, mn in paramBounds], stepsize=2,\n",
    "                                      minimizer_kwargs=dict(args=(mcds, sampDataSet, sampAbbrev, keyFn, adjSer)))\n",
    "dOptRes['bh']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FieldDataSet, MonoCategoryDataSet dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform a multi-Category sightings set into an equivalent mono-Category sightings set,\n",
    "# that is where no sightings has more that one category with positive count (keeping the same total counts).\n",
    "# Ex: A sightings set with 2 Category count columns nMales and nFemales\n",
    "#     * in the input set, you may have 1 sightings with nMales = 5 and nFemales = 2\n",
    "#     * in the output set, this sightings have been separated in 2 distinct ones (all other properties left untouched) :\n",
    "#       the 1st with nMales = 5 and nFemales = 0, the 2nd with nMales = 0 and nFemales = 2.\n",
    "\n",
    "# A slower version or ads.separateMultiCategoryCounts :\n",
    "#  from 9.5s to 0.1s with countColumns = [nMalAd1,nAutAd10,nJuv10,nDetTot10,nMalAd5,nAutAd5,nJuv5,nDetTot5,nTotAd5,nTotAd10]\n",
    "#  on the \"ACDC 2019 Naturalist\" multi-Category data set (~4000 rows)\n",
    "def separateMultiCategoryCounts_slow_version(dfInSightings, countColumns):\n",
    "    \n",
    "    outSightings = list()\n",
    "\n",
    "    for lbl, sInSight in dfInSightings.iterrows():\n",
    "        \n",
    "        # [a little optimisation ?] If this is already a mono-Category sightings, simply append it as is.\n",
    "        sCounts = sInSight[countColumns]\n",
    "        sCounts = sCounts[sCounts > 0]\n",
    "        if len(sCounts) == 1:\n",
    "            \n",
    "            outSightings.append(sInSight)\n",
    "            \n",
    "            continue\n",
    "\n",
    "        # If it is a multi-Category sightings, we need to split it down.\n",
    "        for col in sCounts.index:\n",
    "\n",
    "            sOutSight = sInSight.copy()\n",
    "            sOutSight[countColumns] = 0\n",
    "            sOutSight[col] = sInSight[col]\n",
    "\n",
    "            outSightings.append(sOutSight)\n",
    "\n",
    "    return pd.DataFrame(data=outSightings, index=np.arange(len(outSightings)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObs = pd.read_csv('refin/ACDC2019-Naturalist-ExtraitObsBrutesAvecDist.txt', sep='\\t', decimal=',')\n",
    "dfObs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countCols =  ['nMalAd10', 'nAutAd10', 'nMalAd5', 'nAutAd5']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sCounts = dfObs[countCols].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfObs), sCounts.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(dfObs) == 724\n",
    "assert not any(sCounts - pd.Series({'nMalAd10': 613, 'nAutAd10': 192, 'nMalAd5': 326, 'nAutAd5': 102}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfObsMonoCat_slow = separateMultiCategoryCounts_slow_version(dfObs, countCols)\n",
    "len(dfObsMonoCat_slow), dfObsMonoCat_slow[countCols].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform a multi-individual mono-Category sightings set into an equivalent mono-individual mono-Category sightings set\n",
    "# that is where no sightings has more that one individual per category (keeping the same total counts).\n",
    "# Ex: A sightings set with 2 mono-Category count columns nMales and nFemales\n",
    "#     In input set, you may have 1 sightings with nMales = 3 and nFemales = 0 (but none with nMales and nFemales > 0)\n",
    "#     In out set, no : this sightings have been separated in 3 distinct ones (all other properties left untouched) :\n",
    "#                      all with nMales = 1 and nFemales = 0.\n",
    "\n",
    "# A slower version or ads.individualiseMonoCategoryCounts :\n",
    "#  from 15.5s to 0.06s with countColumns = [nMalAd1,nAutAd10,nJuv10,nDetTot10,nMalAd5,nAutAd5,nJuv5,nDetTot5,nTotAd5,nTotAd10]\n",
    "#  on the \"ACDC 2019 Naturalist\" mono-Category data set (~20000 rows)\n",
    "def individualiseMonoCategoryCounts_slow(dfInSightings, countColumns):\n",
    "\n",
    "    \n",
    "    outSightings = list()\n",
    "\n",
    "    for lbl, sInSight in dfInSightings.iterrows():\n",
    "\n",
    "        # [a little check] Multi-Category sightings not supported here.\n",
    "        sCounts = sInSight[countColumns]\n",
    "        sCounts = sCounts[sCounts > 0]\n",
    "        assert len(sCounts) == 1, 'Error: Multi-Category sightings not supported ' + str(lbl, sInSight)\n",
    "        \n",
    "        # Get the positive count column and its value\n",
    "        posCol = sCounts.index[0]\n",
    "        count = sCounts[posCol]\n",
    "\n",
    "        # [a little optimisation ?] If this is a mono-individual sightings, simply append it as is.\n",
    "        if count == 1:\n",
    "            \n",
    "            outSightings.append(sInSight)\n",
    "\n",
    "            continue\n",
    "\n",
    "        # If it is a multi-individual sightings, we need to split it down.\n",
    "        while count > 0:\n",
    "\n",
    "            sOutSight = sInSight.copy()\n",
    "            sOutSight[posCol] = 1\n",
    "\n",
    "            outSightings.append(sOutSight)\n",
    "\n",
    "            count -= 1\n",
    "\n",
    "    return pd.DataFrame(data=outSightings, index=np.arange(len(outSightings)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfObsIndiv_slow = individualiseMonoCategoryCounts_slow(dfObsMonoCat_slow, countCols)\n",
    "len(dfObsIndiv_slow), dfObsIndiv_slow[countCols].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform a multi-individual multi-Category sightings set into an equivalent mono-individual multi-Category sightings set\n",
    "# that is where no sightings has more that one individual per category (keeping the same total counts).\n",
    "# Ex: A sightings set with 2 Category count columns nMales and nFemales\n",
    "#     In input set, you may have 1 sightings with nMales = 5 and nFemales = 2\n",
    "#     In out set, no : this sightings have been separated in 5 distinct ones (all other properties left untouched) :\n",
    "#                      the 2 1st ones with nMales = 1 and nFemales = 1, the last 3 ones with nMales = 1 and nFemales = 0.\n",
    "\n",
    "# Finally, of no use : simply chain ads.separateMultiCategoryCounts and ads.individualiseMonoCategoryCounts\n",
    "# And from far much slower !\n",
    "def individualiseMultiCategoryCounts(dfInSightings, countColumns):\n",
    "    \n",
    "    outSightings = list()\n",
    "\n",
    "    for lbl, sInSight in dfInSightings.iterrows():\n",
    "\n",
    "        # [a little optimisation ?] If this is a mono-individual sightings, simply append it as is.\n",
    "        sCounts = sInSight[countColumns]\n",
    "        if sCounts.max() == 1:\n",
    "            \n",
    "            outSightings.append(sInSight)\n",
    "            \n",
    "            continue\n",
    "\n",
    "        # If it is a multi-individual sightings, we need to split it down.\n",
    "        sCounts = sCounts.copy()\n",
    "\n",
    "        while sCounts.max() > 0:\n",
    "\n",
    "            sOutSight = sInSight.copy()\n",
    "            sOutSight[countColumns] = sCounts.apply(lambda n: 1 if n > 0 else 0)\n",
    "\n",
    "            outSightings.append(sOutSight)\n",
    "\n",
    "            sCounts = sCounts.apply(lambda n: n-1 if n > 0 else 0)\n",
    "\n",
    "    return pd.DataFrame(data=outSightings, index=list(range(len(outSightings))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add \"abscence\" sightings to field data collected on transects for a given sample\n",
    "# Warning: A special version for an all-taxon data set\n",
    "# * dfInSights : input data table\n",
    "# * transectCol : the name of the transect id column\n",
    "# * taxonSampleCol : the name of the taxon id column\n",
    "# * otherSampleCols : the names of the other sample id columns (taxon id not included)\n",
    "# * expectedTaxa : the expected taxon ids : absence sightings are there to make sure\n",
    "#                  all of the taxa are found at least once in the output data table\n",
    "def addAbsenceSightings(dfInSights, transectCol, taxonSampleCol, otherSampleCols, expectedTaxa):\n",
    "    \n",
    "    def absenceSightings(taxonCol, taxon, dAbscSightTmpl):\n",
    "        dAbscSight = dAbscSightTmpl.copy()\n",
    "        dAbscSight[taxonCol] = taxon\n",
    "        return dAbscSight\n",
    "\n",
    "    assert not dfInSights.empty, 'Error : Empty sightings data to complete !'\n",
    "\n",
    "    ldfAbscSights = list()\n",
    "\n",
    "    # Use 1st sightings of the sample to build the absence sightings prototype\n",
    "    # (all null columns except for the sample identification ones)\n",
    "    dAbscSightTmpl = dfInSights.iloc[0].to_dict()\n",
    "    dAbscSightTmpl.update({ k: None for k in dAbscSightTmpl.keys() if k not in otherSampleCols })\n",
    "\n",
    "    # For each transect\n",
    "    for transect in dfInSights[transectCol].unique():\n",
    "\n",
    "        # Update absence sightings template with transect id\n",
    "        dAbscSightTmpl.update({ transectCol: transect })\n",
    "\n",
    "        # Generate the absence sightings from it : 1 per lacking taxon\n",
    "        lackingTaxa = \\\n",
    "          set(expectedTaxa) - set(dfInSights.loc[dfInSights[transectCol] == transect, taxonSampleCol].unique())\n",
    "        dfAbscSights = pd.DataFrame([absenceSightings(taxonSampleCol, txn, dAbscSightTmpl) for txn in lackingTaxa])\n",
    "\n",
    "        # Save the data frame for later\n",
    "        ldfAbscSights.append(dfAbscSights)\n",
    "\n",
    "    # Concat all absence data frames into one.\n",
    "    dfOutSights = pd.concat([dfInSights] + ldfAbscSights)\n",
    "\n",
    "    # Reset index (for unique labels).\n",
    "    dfOutSights.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    # Done.\n",
    "    return dfOutSights\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tests unitaires de addAbsenceSightings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define transect, taxon and sample columns\n",
    "transectCol = 'Point'\n",
    "taxonCol = 'Espece'\n",
    "sampleCols = ['Passage', 'Adulte', 'Duree']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The set of expected taxa ... of which we'll look for abscence on every location\n",
    "expectedTaxa = list(dfObsIndiv[taxonCol].unique())\n",
    "\n",
    "assert len(expectedTaxa) == 58\n",
    "\n",
    "', '.join(expectedTaxa), len(expectedTaxa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select 1 random sample\n",
    "passage = 'a'\n",
    "adulte = 'm'\n",
    "duree = '10'\n",
    "dfObsIndivSmpl = dfObsIndiv[(dfObsIndiv.Passage == passage) & (dfObsIndiv.Adulte == adulte) & (dfObsIndiv.Duree == duree)]\n",
    "\n",
    "assert len(dfObsIndivSmpl) == 322 and dfObsIndivSmpl[transectCol].nunique() == 21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfObsIndivAbscSmpl = ads.addAbsenceSightings(dfObsIndivSmpl, transectCol, taxonCol, expectedTaxa, sampleCols)\n",
    "len(dfObsIndivAbscSmpl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check for no change in sample columns\n",
    "assert list(dfObsIndivAbscSmpl.columns) == list(dfObsIndivSmpl.columns)\n",
    "\n",
    "# Check for number of added rows\n",
    "assert len(dfObsIndivAbscSmpl) == 1333\n",
    "\n",
    "# Check for no change in number of transect and taxa\n",
    "assert dfObsIndivAbscSmpl[transectCol].nunique() == 21 and dfObsIndivAbscSmpl[taxonCol].nunique() == 58\n",
    "\n",
    "# Check for no change in sample identification\n",
    "assert list(dfObsIndivAbscSmpl.Passage.unique()) == [passage]\n",
    "assert list(dfObsIndivAbscSmpl.Adulte.unique()) == [adulte]\n",
    "assert list(dfObsIndivAbscSmpl.Duree.unique()) == [duree]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndivSmpl.sort_values(by=['Passage', 'Observateur', 'Point', 'Espece', 'distMem']).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndivAbscSmpl.sort_values(by=['Passage', 'Observateur', 'Point', 'Espece', 'distMem']).head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Performance test\n",
    "print('Passage  Adulte Duree NbDonnees')\n",
    "\n",
    "for passage in ['a', 'b', 'a+b']: \n",
    "    \n",
    "    for adulte in ['m', 'a', 'm+a']:\n",
    "    \n",
    "        for duree in ['5', '10']:\n",
    "            \n",
    "            passages = passage.split('+')\n",
    "            adultes = adulte.split('+')\n",
    "            dfObsIndivSmpl = dfObsIndiv[dfObsIndiv.Passage.isin(passages) & dfObsIndiv.Adulte.isin(adultes) \\\n",
    "                                        & (dfObsIndiv.Duree == duree)]\n",
    "            \n",
    "            dfObsIndivAbscSmpl = ads.addAbsenceSightings(dfObsIndivSmpl, transectCol, taxonCol, expectedTaxa, sampleCols)\n",
    "            \n",
    "            print(passage, adulte, duree, ':', len(dfObsIndivAbscSmpl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResultsSet.append\n",
    "\n",
    "Updated version thanks to [pd.DataFrame.append(pd.Series) study](#Appending-series-to-DataFrame-...-columns-order) below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def append(dfData, sdfResult, sCustomHead):\n",
    "\n",
    "    if sCustomHead is not None:\n",
    "        if isinstance(sdfResult, pd.Series):\n",
    "            sdfResult = sCustomHead.append(sdfResult)\n",
    "        else: # DataFrame\n",
    "            dfCustomHead = pd.DataFrame([sCustomHead]*len(sdfResult)).reset_index(drop=True)\n",
    "            sdfResult = pd.concat([dfCustomHead, sdfResult], axis='columns')\n",
    "\n",
    "    # Normal append if _dfData not empty ; otherwise initialise _dfData in a way\n",
    "    # that keeps the original types of sdfResult / \n",
    "    if dfData.columns.empty:\n",
    "        if isinstance(sdfResult, pd.Series):\n",
    "            dfData = pd.DataFrame([sdfResult])\n",
    "        else: # DataFrame\n",
    "            dfData = sdfResult\n",
    "    else:\n",
    "        dfData = dfData.append(sdfResult, ignore_index=True)\n",
    "\n",
    "    return dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Initialise DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Empty\n",
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not empty, mono-index columns\n",
    "df = pd.DataFrame([dict(a=1, b=2.5, c='x', x='a', y=1, z=1.78),\n",
    "                   dict(a=2, b=4.5, c='y', x='b', y=2, z=5.88889)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not empty, multi-index columns\n",
    "df = pd.DataFrame([{('a', 'z'): 1, ('b', 'y'): 2.5, ('c', 'x'): 'x', ('x', 'w'): 'a', ('y', 'v'): 1, ('z', 'u'): 1.78},\n",
    "                   {('a', 'z'): 2, ('b', 'y'): 4.5, ('c', 'x'): 'y', ('x', 'w'): 'b', ('y', 'v'): 2, ('z', 'u'): 5.88889}])\n",
    "df.columns = pd.MultiIndex.from_tuples(df.columns)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Initialise Series / DataFrame to append"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mono-index\n",
    "sh = pd.Series(dict(a=3, b=5.978, c='w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sr = pd.Series(dict(x='c', y=4, z=9.567))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sr = pd.DataFrame([dict(x='d', y=9, z=12.9),\n",
    "                   dict(x='e', y=8, z=7.778)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Multi-index\n",
    "sh = pd.Series({('a', 'z'): 3, ('b', 'y'): 5.978, ('c', 'x'): 'w'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sr = pd.Series({('x', 'w'): 'c', ('y', 'v'): 4, ('z', 'u'): 9.567})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sr = pd.DataFrame([{('x', 'w'): 'd', ('y', 'v'): 9, ('z', 'u'): 12.9},\n",
    "                   {('x', 'w'): 'e', ('y', 'v'): 8, ('z', 'u'): 7.778}])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. append Series /DataFrame to DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = append(df, sr, sh)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. See what's happening"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MCDSAnalysisResultsSet quality indicators functions dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.graph_objs as plygo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normNTotPars(nTotPars, a=0.2, b=0.6, c=2, d=1):\n",
    "        #return 1 / (a * sRes[cls.CLNTotPars] + b)  # Trop pénalisant: a=0.2, b=1\n",
    "        return 1 / (a * max(c, nTotPars)**d + b)  # Mieux: a=0.2, b=0.6 / a=0.2, b=0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xDomain = [0, 1, 2, 3, 4, 5, 6, 8, 10 , 12]\n",
    "paramSets = [dict(a=0.2, b=0.6, c=2, d=1), dict(a=0.2, b=0.8, c=1, d=1), dict(a=0.3, b=0.7, c=1, d=1),\n",
    "             dict(a=0.2, b=0.8, c=1, d=2), dict(a=0.2, b=0.8, c=1, d=1.5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plygo.Figure()\n",
    "\n",
    "for params in paramSets:\n",
    "    fig.add_trace(plygo.Scatter(x=xDomain,\n",
    "                                y=[normNTotPars(nTotPars, **params) for nTotPars in xDomain],\n",
    "                                name=str(params)))\n",
    "\n",
    "fig.update_layout(title='NTotPars normalisation')\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normCVDens(dCv, a=12):\n",
    "        #return max(0, 1 - a * sRes[cls.CLDCv]) # Pas très pénalisant: a=1\n",
    "        return math.exp(-a * dCv ** 2) # Mieux : déjà ~0.33 à 30% (a=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xDomain = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.08, 0.15, 0.30, 0.40, 0.5, 0.7, 1.0]\n",
    "paramSets = [dict(a=8), dict(a=12), dict(a=14), dict(a=16), dict(a=20)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plygo.Figure()\n",
    "\n",
    "for params in paramSets:\n",
    "    fig.add_trace(plygo.Scatter(x=xDomain,\n",
    "                                y=[normCVDens(cvDens, **params) for cvDens in xDomain],\n",
    "                                name=str(params)))\n",
    "\n",
    "fig.update_layout(title='CVDens normalisation')\n",
    "\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis results filtering and sorting tools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Adaptative filtering\n",
    "\n",
    "See [devarchives2.ipynb / Development : Automated filtering and sorting of optanalysis results](./devarchives2.ipynb#Development-%3A-Automated-filtering-and-sorting-of-optanalysis-results) needed input data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nTgtRmdr = 5\n",
    "chi2Range = np.linspace(start=0.8, stop=0.1, num=8)\n",
    "\n",
    "for lblEch, sEch in dfStatsEch.iterrows():\n",
    "\n",
    "    dfFSEchRep = dfFilSorRep[dfFilSorRep.Echant == lblEch]\n",
    "    #print('#{} {} : {}'.format(lblEch, sEch['Abréviation'], len(dfFSEchRep)), end=' => ')\n",
    "\n",
    "    for chi2 in chi2Range:\n",
    "        i2DropEch = dfFSEchRep[dfFSEchRep['Chi2 P'] < chi2].index.to_list()\n",
    "        #print(len(i2DropEch), end=', ')\n",
    "        if len(dfFSEchRep) - len(i2DropEch) >= nTgtRmdr:\n",
    "            break\n",
    "\n",
    "    #print(' => ', chi2, len(i2DropEch))\n",
    "    dfFilSorRep.drop(i2DropEch, inplace=True)\n",
    "\n",
    "len(dfFilSorRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterAdaptScheme(dfRes, sampleIds, sampleIdCol, critCol, ascendCrit=True, nMaxSteps=5, nMinRes=10):\n",
    "    \n",
    "    \"\"\"Fonction générique de filtrage avec stratégie de contrôle du nombre de résultats conservé\n",
    "    via un schéma adaptatif de seuillage sur 1 critère (fonction de son domaine réel de valeurs)\n",
    "    \"\"\"\n",
    "    \n",
    "    assert nMaxSteps > 2, 'At least 3 steps are mandatory'\n",
    "    \n",
    "    # For each sample ...\n",
    "    i2Drop = []\n",
    "    for sampId in sampleIds:\n",
    "\n",
    "        # Extract results.\n",
    "        dfSampRes = dfRes[dfRes[sampleIdCol] == sampId]\n",
    "        print('#{}: {} results'.format(sampId, len(dfSampRes)), end=' => ')\n",
    "\n",
    "        # Compute criteria threshold variation scheme from actual value domain\n",
    "        start = dfSampRes[critCol].max() if ascendCrit else dfSampRes[critCol].min()\n",
    "        stop = dfSampRes[critCol].min() if ascendCrit else dfSampRes[critCol].max()\n",
    "        print(f'{start=:.3f} {stop=:.3f}', end=': ')\n",
    "\n",
    "        # For each step of the scheme ...\n",
    "        for thresh in np.linspace(start, stop, num=nMaxSteps)[1:-1]:\n",
    "\n",
    "            # Try and apply the threshold step : number of dropped results if ...\n",
    "            if ascendCrit:\n",
    "                i2DropSamp = dfSampRes[dfSampRes[critCol] < thresh].index\n",
    "            else:\n",
    "                i2DropSamp = dfSampRes[dfSampRes[critCol] > thresh].index\n",
    "            \n",
    "            # Stop if we are above the minimum number of results.\n",
    "            print('t={:.3f}/k={}'.format(thresh, len(dfSampRes) - len(i2DropSamp)), end=', ')\n",
    "            if len(dfSampRes) - len(i2DropSamp) >= nMinRes:\n",
    "                break\n",
    "\n",
    "        print('done')\n",
    "\n",
    "        # Append index to drop for sample to the final one\n",
    "        i2Drop = i2DropSamp if not len(i2Drop) else i2Drop.append(i2DropSamp)\n",
    " \n",
    "    return i2Drop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Close distance truncations grouping from report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSelRep = dfRep[dfRep.Echant == 0].copy()\n",
    "\n",
    "epsilDist = 1e-6\n",
    "\n",
    "#truncCol, minIntrvDist, maxIntrvLen = 'Dist Tronc Drte', 25.0, 25.0\n",
    "truncCol, minIntrvDist, maxIntrvLen = 'Dist Tronc Gche', 5.0, 5.0\n",
    "\n",
    "dfIntrv = dfSelRep[[truncCol]].dropna().sort_values(by=truncCol)\n",
    "\n",
    "dfIntrv['deltaDist'] = dfIntrv[truncCol].diff()\n",
    "dfIntrv.loc[dfIntrv[truncCol].idxmin(), 'deltaDist'] = np.inf\n",
    "\n",
    "dfIntrv.dropna(inplace=True)\n",
    "dfIntrv = dfIntrv[dfIntrv.deltaDist > 0]\n",
    "\n",
    "dfIntrv['dMin'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, truncCol]\n",
    "dfIntrv['dSup'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, truncCol].shift(-1).dropna()\n",
    "dfIntrv.loc[dfIntrv['dMin'].idxmax(), 'dSup'] = np.inf\n",
    "dfIntrv.dropna(inplace=True)\n",
    "\n",
    "sSelDist = dfSelRep[truncCol]\n",
    "dfIntrv['dSup'] = dfIntrv['dSup'].apply(lambda supV: sSelDist[sSelDist < supV].max() + epsilDist)\n",
    "\n",
    "dfIntrv = dfIntrv[['dMin', 'dSup']].reset_index(drop=True)\n",
    "dfIntrv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsNewIntrvs = list()\n",
    "for _, sIntrv in dfIntrv.iterrows():\n",
    "    \n",
    "    if sIntrv.dSup - sIntrv.dMin > maxIntrvLen:\n",
    "        nSubIntrvs = (sIntrv.dSup - sIntrv.dMin) / maxIntrvLen\n",
    "        nSubIntrvs = int(nSubIntrvs) if nSubIntrvs - int(nSubIntrvs) < 0.5 else int(nSubIntrvs) + 1\n",
    "        subIntrvLen = (sIntrv.dSup - sIntrv.dMin) / nSubIntrvs\n",
    "        lsNewIntrvs += [pd.Series(dict(dMin=sIntrv.dMin + i * subIntrvLen, \n",
    "                                       dSup=min(sIntrv.dMin + (i + 1) * subIntrvLen, sIntrv.dSup)))\n",
    "                        for i in range(nSubIntrvs)]\n",
    "    else:\n",
    "        lsNewIntrvs.append(sIntrv)\n",
    "        \n",
    "dfIntrv = pd.DataFrame(lsNewIntrvs).reset_index(drop=True)\n",
    "dfIntrv.sort_values(by='dMin', inplace=True)\n",
    "dfIntrv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSelRep['Grp ' + truncCol] = \\\n",
    "    dfSelRep[truncCol].apply(lambda d: 0 if pd.isnull(d) else 1 + dfIntrv[(dfIntrv.dMin <= d) & (dfIntrv.dSup > d)].index[0])\n",
    "dfSelRep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSelRep['Grp ' + truncCol].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSelRep.sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Close distance truncations grouping from results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results: En provenance de \"Visionature-ds-point.ipynb/XVI. Analyses automatiques\"\n",
    "resultsCopy = results.copy()\n",
    "self = results\n",
    "self._dfData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "self._dfData.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "self.miSampleCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsDist = 1e-6\n",
    "\n",
    "ldTruncIntrvSpecs = [dict(col=self.CLParTruncLeft, minDist=5.0, maxLen=5.0),\n",
    "                     dict(col=self.CLParTruncRight, minDist=25.0, maxLen=25.0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "self.miSampleCols.append(pd.MultiIndex.from_tuples([self.sampleIndCol]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Les échantillons\n",
    "dfSamples = self._dfData[pd.MultiIndex.from_tuples([self.sampleIndCol]).append(self.miSampleCols)].drop_duplicates()\n",
    "dfSamples.set_index(self.sampleIndCol, inplace=True)\n",
    "assert len(dfSamples) == dfSamples.index.nunique()\n",
    "\n",
    "dfSamples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "self.CLCAFilSor = 'auto filter sort'\n",
    "self.CLTTruncGroup = 'Group'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pour chaque échantillon,\n",
    "for lblSamp, sSamp in dfSamples.iterrows():\n",
    "    \n",
    "    print('#{} {} :'.format(lblSamp, ','.join([f'{k[1]}:{v}' for k, v in sSamp.items()])))\n",
    "\n",
    "    # Sélectionner les résultats associés, et uniquement ceux-là\n",
    "    dfSampRes = self._dfData[self._dfData[self.sampleIndCol] == lblSamp]\n",
    "\n",
    "    # Pour chaque type de troncature (optimisée ou non),\n",
    "    for isOpt in sorted(dfSampRes[self.optimTruncFlagMCol].unique()):\n",
    "        \n",
    "        print('* {}optimised'.format('' if isOpt else 'non ').title(), end=' : ')\n",
    "\n",
    "        # Sélectionner les résultats associés, et uniquement ceux-là\n",
    "        dfSampResPerOpt = dfSampRes[dfSampRes[self.optimTruncFlagMCol] == isOpt]\n",
    "\n",
    "        for dTrunc in ldTruncIntrvSpecs:\n",
    "\n",
    "            truncCol = dTrunc['col']\n",
    "            minIntrvDist = dTrunc['minDist']\n",
    "            maxIntrvLen = dTrunc['maxLen']\n",
    "\n",
    "            print(truncCol[1], end=', ')\n",
    "\n",
    "            sSelDist = dfSampResPerOpt[truncCol]\n",
    "            dfIntrv = pd.DataFrame(dict(dist=sSelDist.dropna().sort_values().values))\n",
    "\n",
    "            # Ecarts non nuls de distances entre distances consécutives triées\n",
    "            dfIntrv['deltaDist'] = dfIntrv.dist.diff()\n",
    "            dfIntrv.loc[dfIntrv.dist.idxmin(), 'deltaDist'] = np.inf\n",
    "            dfIntrv.dropna(inplace=True)\n",
    "            dfIntrv = dfIntrv[dfIntrv.deltaDist > 0].copy()\n",
    "\n",
    "            # Début et fin de chaque intervalle (fermé à gauche = dMin, ouvert à droite = dSup)\n",
    "            dfIntrv['dMin'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, 'dist']\n",
    "            dfIntrv['dSup'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, 'dist'].shift(-1).dropna()\n",
    "            dfIntrv.loc[dfIntrv['dMin'].idxmax(), 'dSup'] = np.inf\n",
    "            dfIntrv.dropna(inplace=True)\n",
    "\n",
    "            dfIntrv['dSup'] = dfIntrv['dSup'].apply(lambda supV: sSelDist[sSelDist < supV].max() + epsDist)\n",
    "\n",
    "            dfIntrv = dfIntrv[['dMin', 'dSup']].reset_index(drop=True)\n",
    "\n",
    "            # Si les intervalles ainsi détectés sont trop larges, on les découpe en tranches égales\n",
    "            lsNewIntrvs = list()\n",
    "            for _, sIntrv in dfIntrv.iterrows():\n",
    "\n",
    "                if sIntrv.dSup - sIntrv.dMin > maxIntrvLen:\n",
    "                    nSubIntrvs = (sIntrv.dSup - sIntrv.dMin) / maxIntrvLen\n",
    "                    nSubIntrvs = int(nSubIntrvs) if nSubIntrvs - int(nSubIntrvs) < 0.5 else int(nSubIntrvs) + 1\n",
    "                    subIntrvLen = (sIntrv.dSup - sIntrv.dMin) / nSubIntrvs\n",
    "                    lsNewIntrvs += [pd.Series(dict(dMin=sIntrv.dMin + nInd * subIntrvLen, \n",
    "                                                   dSup=min(sIntrv.dMin + (nInd + 1) * subIntrvLen, sIntrv.dSup)))\n",
    "                                    for nInd in range(nSubIntrvs)]\n",
    "                else:\n",
    "                    lsNewIntrvs.append(sIntrv)\n",
    "\n",
    "            dfIntrv = pd.DataFrame(lsNewIntrvs).reset_index(drop=True)\n",
    "            dfIntrv.sort_values(by='dMin', inplace=True)\n",
    "\n",
    "            # Attribution du numéro de groupe de troncatures à chaque distance mesurée (0 = pas de troncature)\n",
    "            sb = (self._dfData[self.sampleIndCol] == lblSamp) & (self._dfData[self.optimTruncFlagMCol] == isOpt)\n",
    "            self._dfData.loc[sb, (self.CLCAFilSor, truncCol[1], self.CLTTruncGroup)] = \\\n",
    "                self._dfData.loc[sb, truncCol].apply(lambda d: 0 if pd.isnull(d) \\\n",
    "                                                       else 1 + dfIntrv[(dfIntrv.dMin <= d) & (dfIntrv.dSup > d)].index[0])\n",
    "\n",
    "        print(len(dfSampResPerOpt))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "self._dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Filtering and sorting orders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#class RS(ads.MCDSTruncOptanalysisResultsSet):\n",
    "#    \n",
    "#    CLNObs = 'NObs'\n",
    "#    CLNTotObs = 'NTot Obs'\n",
    "#    CLNTotPars = 'NbTot Pars'\n",
    "#    CLChi2  = 'Chi2 P'\n",
    "#    CLDCv   = 'CoefVar Densité'\n",
    "#    CLKS    = 'KS P'\n",
    "#    CLCvMUw = 'CvM Uw P'\n",
    "#    CLCvMCw = 'CvM Cw P'\n",
    "#    \n",
    "#    def __init__(self):\n",
    "#        pass\n",
    "#\n",
    "#rs = RS()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.AutoFilSorKeySchemes[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DRes2RefRepCols = {v:k for k, v in DRefRep2ResCols.items()}\n",
    "DRes2RefRepCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results\n",
    "lblSamp = 0\n",
    "dfSampRes = dfRes[dfRes.Echant == lblSamp].copy()\n",
    "\n",
    "scheme = results.AutoFilSorKeySchemes[6]\n",
    "print('group' in scheme, scheme)\n",
    "\n",
    "# Sort results\n",
    "dfSampRes.sort_values(by=results.transColumns(scheme['sort'], 'fr'), ascending=scheme['ascend'], \n",
    "                      na_position=scheme.get('napos', 'last'), inplace=True)\n",
    "dfSampRes.set_index('Analyse', inplace=True)\n",
    "\n",
    "# Compute order (specific to groups or global).\n",
    "if 'group' in scheme:\n",
    "    sSampOrder = dfSampRes.groupby(results.transColumns(scheme['group'], 'fr'), dropna=False).cumcount()\n",
    "else:\n",
    "    sSampOrder = pd.Series(data=range(len(dfSampRes)), index=dfSampRes.index)\n",
    "\n",
    "sSampOrder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Old report, new method\n",
    "lblSamp = 0\n",
    "dfSampRep = dfRefRep[dfRefRep.Echant == lblSamp].rename(columns=DRes2RefRepCols).copy()\n",
    "\n",
    "optimTruncCol = 'OptimTrunc'\n",
    "#scheme = dict(name='Meil CKCv Tronc Proch',  # Meilleur Chi2&KS&DCV par groupe de troncatures proches\n",
    "#              sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "#                    'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx'],\n",
    "#              ascend=[True, True, True, False, False, True, False, True],\n",
    "#              group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte'])\n",
    "scheme= dict(name='Meil Qual Chi2 Tronc Proch',  # Meilleur Qualité combinée Chi2+ par groupe de troncatures proches\n",
    "             sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "                   'Qual Chi2'],\n",
    "             ascend=[True, True, True, False],\n",
    "             group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte'])\n",
    "\n",
    "print('group' in scheme, scheme)\n",
    "\n",
    "# Sort results\n",
    "dfSampRep.sort_values(by=scheme['sort'], ascending=scheme['ascend'], \n",
    "                      na_position=scheme.get('napos', 'last'), inplace=True)\n",
    "dfSampRep.set_index('Analyse', inplace=True)\n",
    "\n",
    "# Compute order (specific to groups or global).\n",
    "if 'group' in scheme:\n",
    "    sRepSampOrder = dfSampRep.groupby(scheme['group'], dropna=False).cumcount()\n",
    "else:\n",
    "    sRepSampOrder = pd.Series(data=range(len(dfSampRep)), index=dfSampRep.index)\n",
    "\n",
    "sRepSampOrder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp = sRepSampOrder.to_frame(name='rep').join(sSampOrder.to_frame(name='res')).sort_index()\n",
    "dfComp[dfComp.res != dfComp.rep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reduce float resolution (pb with least significant bits ?)\n",
    "dfSampRep[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "           'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']] = \\\n",
    "    dfSampRep[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "               'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].astype(float)\n",
    "\n",
    "# Sort results\n",
    "dfSampRep = dfSampRep.sort_values(by=scheme['sort'], ascending=scheme['ascend'], \n",
    "                                  na_position=scheme.get('napos', 'last'))\n",
    "\n",
    "# Compute order (specific to groups or global).\n",
    "sRepSampOrder = dfSampRep.groupby(scheme['group'], dropna=False).cumcount() \\\n",
    "                 if 'group' in scheme else range(len(dfSampRep))\n",
    "\n",
    "sRepSampOrder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSampRes[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "           'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']] = \\\n",
    "    dfSampRes[['OptimTrunc', 'Groupe Tronc Gche', 'Groupe Tronc Drte',\n",
    "               'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].astype(float)\n",
    "\n",
    "# Sort results\n",
    "dfSampRes = dfSampRes.sort_values(by=results.transColumns(scheme['sort'], 'fr'), ascending=scheme['ascend'], \n",
    "                                  na_position=scheme.get('napos', 'last'))\n",
    "\n",
    "# Compute order (specific to groups or global).\n",
    "sSampOrder = dfSampRes.groupby(results.transColumns(scheme['group'], 'fr'), dropna=False).cumcount() \\\n",
    "             if 'group' in scheme else range(len(dfSampRes))\n",
    "\n",
    "sSampOrder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp = sRepSampOrder.to_frame(name='rep').join(sSampOrder.to_frame(name='res')).sort_index()\n",
    "dfComp[dfComp.res != dfComp.rep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSampRep[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "           'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].to_excel('tmp/_.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSampRes[['OptimTrunc', 'Groupe Tronc Gche', 'Groupe Tronc Drte',\n",
    "           'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].to_excel('tmp/__.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSampRes[['OptimTrunc', 'Groupe Tronc Gche', 'Groupe Tronc Drte',\n",
    "           'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].rename(columns=DRes2RefRepCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSampRep[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "           'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp2 = dfSampRes[['OptimTrunc', 'Groupe Tronc Gche', 'Groupe Tronc Drte',\n",
    "                     'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']] \\\n",
    "             .rename(columns=DRes2RefRepCols).sort_index() \\\n",
    "             .compare(dfSampRep[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "                                 'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].sort_index())\n",
    "dfComp2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp2[('Chi2 P', 'self')] - dfComp2[('Chi2 P', 'other')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp3 = dfSampRes[['OptimTrunc', 'Groupe Tronc Gche', 'Groupe Tronc Drte',\n",
    "                     'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].astype(np.float32) \\\n",
    "  .rename(columns=DRes2RefRepCols).sort_index() \\\n",
    "  .compare(dfSampRep[['OptimTrunc', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "                      'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx']].astype(np.float32).sort_index())\n",
    "dfComp3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp4 = dfSampRes[ordDiffCols].sort_index().compare(dfSampRep.rename(columns=DRefRep2ResCols)[ordDiffCols].sort_index())\n",
    "dfComp4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis elapsed time statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[col for col in results.columns if col[0] == 'run output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subsetCols = [('parameters', 'estimator key function', 'Value'),\n",
    "              ('parameters', 'estimator adjustment series', 'Value'),   \n",
    "              ('run output', 'elapsed time', 'Value'),\n",
    "              ('run output', 'run status', 'Value'),\n",
    "              ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "              ('detection probability', 'number of key function parameters (NKP)', 'Value'),\n",
    "              ('detection probability', 'number of adjustment term parameters (NAP)', 'Value'),\n",
    "              ('run output', 'run folder', 'Value')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = results.dfTransData('fr', columns=subsetCols)\n",
    "#df[['NObs', 'NbPars FnClé', 'NbPars SérAjust']] = df[['NObs', 'NbPars FnClé', 'NbPars SérAjust']].astype(int)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['NObs'].describe(), df['DuréeExec'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.histogram(df['DuréeExec'], bins=80, range=(0, 400))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df['DuréeExec'] > 60].DossierExec.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df['DuréeExec'] < 60, ['NObs', 'DuréeExec']].plot.scatter(y='NObs', x='DuréeExec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# II. Python recipes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function parameters discovery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import inspect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Base(object):\n",
    "    def __init__(self, a, b, xa=1, xb=2):\n",
    "        frame = inspect.currentframe()\n",
    "        args, _, _, values = inspect.getargvalues(frame)\n",
    "        print(args, values)\n",
    "        print('function name \"{}\"'.format(inspect.getframeinfo(frame)[2]))\n",
    "        for i in args:\n",
    "            print('    {} = {}'.format(i, values[i]))\n",
    "        print([(i, values[i]) for i in args])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = Base(4, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Named tuple from dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import namedtuple as ntuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = dict(a=1, b=[3, 2], c='xxx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NT = ntuple('NT', d.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nt = NT(**d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Appending series to series ... index order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=pd.MultiIndex.from_tuples([('B', 'b'), ('B', 'a'), ('A', 'c')]), data=[1, 2, 3], name=0)\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.append(pd.Series(index=[('A', 'b'), ('A', 'a'), ('B', 'c')], data=[1, 2, 3], name=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Appending series to DataFrame ... columns order"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Append"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=pd.MultiIndex.from_tuples([('B', 'b'), ('B', 'a'), ('A', 'c')]), data=[1, 2, 3], name=0)\n",
    "#df = df.append(s, ignore_index=False) # => df.columns pas MultiIndex !\n",
    "df = df.append([s], ignore_index=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=[('A', 'c'), ('B', 'b'), ('B', 'a')], data=[4, 5, 6], name=1)  # Mêmes colonnes : append ne retrie pas\n",
    "#s = pd.Series(index=[('A', 'a'), ('A', 'b'), ('B', 'c')], data=[4, 5, 6], name=1)  # Nouvelle colonne : append retrie\n",
    "df = df.append([s], ignore_index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=[('A', 'a'), ('B', 'c')], data=[7, 8])\n",
    "df = df.append(s, ignore_index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=[], data=[])\n",
    "df = df.append([s], ignore_index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=[('C', 'd')], data=[9])\n",
    "df = df.append([s], ignore_index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=[('d',)], data=[10])\n",
    "df = df.append(s, ignore_index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Concat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=pd.MultiIndex.from_tuples([('B', 'b'), ('B', 'a'), ('A', 'c')]), data=[1, 2, 3], name=0)\n",
    "df = pd.concat([df, s], axis='columns')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(index=[('B', 'b'), ('B', 'a'), ('A', 'c')], data=[4, 5, 6], name=1) # Mêmes colonnes : concat ne retrie pas\n",
    "#s = pd.Series(index=[('A', 'a'), ('A', 'b'), ('B', 'c')], data=[4, 5, 6], name=1) # Nouvelle colonne : concat retrie\n",
    "df = pd.concat([df, s], axis='columns')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Restore desired columns\n",
    "\n",
    "* desired order,\n",
    "* desired list of columns : new ones, and / or ignored ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add new A/b, D/a and remove B/c and C/d\n",
    "i = pd.MultiIndex.from_tuples([('A', 'c'), ('A', 'b'), ('A', 'a'), ('B', 'b'), ('B', 'a'), ('D', 'a')])\n",
    "i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep added columns (with no data inside)\n",
    "df2 = df.reindex(i, axis='columns')\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove added columns (with no data inside)\n",
    "df2 .dropna(how='all', axis='columns')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Appending partially-columned DataFrame to DataFrame\n",
    "\n",
    "with generation of lacking columns by duplicating a series = a row template"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([dict(a=1, b=2, c=3), dict(a=3, b=4, c=5), dict(a=4, b=5, c=6)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(dict(x=0, y=1), name=9)\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame([s]*len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([pd.DataFrame([s]*len(df)).reset_index(drop=True), df], axis='columns')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=[(1, 2, 3), (4, 5, 6), (7, 5, 6)],\n",
    "                  columns=pd.MultiIndex.from_tuples([('a', 'b'), ('a', 'c'), ('b', 'd')]))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[0, ('a', 'b')] = 9\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OK: no need for passing a MultiIndex to []\n",
    "df[[('a', 'c'), ('b', 'd')]] = 9\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Neither to duplicated ...\n",
    "df[df.duplicated(subset=[('a', 'c'), ('b', 'd')])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assymetric index and columns indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([dict(a=3, b=3, c=2), dict(a=3, b=3, c=3), dict(a=2, b=3, c=3)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df == 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfb = df.applymap(lambda v: v == 3)\n",
    "dfb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label boolean indexing : easy !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfb.all(axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# It works ...\n",
    "df[dfb.all(axis='columns')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# ... and it's fast\n",
    "df[dfb.all(axis='columns')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping only works this more classical way...\n",
    "df.drop(index=df[dfb.all(axis='columns')].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# ... and it's fast\n",
    "df.drop(index=df[dfb.all(axis='columns')].index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns boolean indexing : unsymetric API (as of pandas 1.3) !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfb.all(axis='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# It works this tortuous way, ...\n",
    "df.T[dfb.all(axis='index')].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# ... but it's 3 times slower than for label indexing !\n",
    "df.T[dfb.all(axis='index')].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# It works also this more classical way, ...\n",
    "df[[col for col, b in dfb.all(axis='index').items() if b]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# ... but it's also 3 times slower than for label indexing !\n",
    "df[[col for col, b in dfb.all(axis='index').items() if b]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.T.loc[df.T[dfb.all(axis='index')].index].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.T[dfb.all(axis='index')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping works this classical and tortuous way also ...\n",
    "df.drop(columns=df.T[dfb.all(axis='index')].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# ... but it's 50% slower than the following even more classical way\n",
    "df.drop(columns=df.T[dfb.all(axis='index')].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping works this even more classical way also ...\n",
    "df.drop(columns=[col for col, b in dfb.all(axis='index').items() if b])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# ... and it's quite fast\n",
    "df.drop(columns=[col for col, b in dfb.all(axis='index').items() if b])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check python derivation and class methods / attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Base(object):\n",
    "    \n",
    "    A = 'Base.A'\n",
    "    B = 'Base.B'\n",
    "    \n",
    "    def f(self):\n",
    "        print('Base.f')\n",
    "        return self.g()\n",
    "        \n",
    "    def g(self):\n",
    "        print('Base.g')\n",
    "        return self.A\n",
    "    \n",
    "    def i(self):\n",
    "        print('Base.i')\n",
    "        \n",
    "class Derived(Base):\n",
    "    \n",
    "    A = 'Derived.A'\n",
    "    \n",
    "    @classmethod\n",
    "    def h(cls):\n",
    "        print('Derived.h: A=', cls.A)\n",
    "        return cls.A\n",
    "\n",
    "    def g(self):\n",
    "        print('Derived.g')\n",
    "        return self.h()\n",
    "\n",
    "    def i(self):\n",
    "        print('Derived.i')\n",
    "        super().i()\n",
    "        \n",
    "d = Derived()\n",
    "\n",
    "assert d.f() == 'Derived.A'\n",
    "\n",
    "print('d.B=', d.B)\n",
    "\n",
    "d.i()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An other one\n",
    "class A(object):\n",
    "    X = 5\n",
    "    def __init__(self, y):\n",
    "        self.xy = y * self.X\n",
    "    \n",
    "class B(A):\n",
    "    X = 10\n",
    "    def __init__(self, y):\n",
    "        super().__init__(y=y)\n",
    "\n",
    "a = A(y=2)\n",
    "print(a.X, a.xy)\n",
    "        \n",
    "b = B(y=3)\n",
    "print(b.X, b.xy)\n",
    "\n",
    "print(A.X, B.X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pd.DataFrame.round()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([dict(a=1.00, b=2.00, c=3.00),\n",
    "                   dict(a=1.05, b=2.01, c=3.01), \n",
    "                   dict(a=1.01, b=1.94, c=3.02), \n",
    "                   dict(a=1.09, b=2.00, c=3.00)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfr = df.round(1)\n",
    "dfr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[~dfr.duplicated(keep='last')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.round(decimals=dict(a=1, b=2, c=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.insert(df.columns.get_loc('c'), 'x', np.nan)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.a.where(df.a < 1.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fillna(inplace=True) on column subset\n",
    "\n",
    "Why doesn't it exist ? (pandas <= 1.2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([dict(a=1.00,   b=2.00,   c=3.00),\n",
    "                   dict(a=1.05,   b=np.nan, c=3.01), \n",
    "                   dict(a=np.nan, b=1.94,   c=np.nan), \n",
    "                   dict(a=1.09,   b=np.nan, c=3.00)])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Does not work, and raises a SettingWithCopyWarning warning with pandas <= 1.2.5 at least\n",
    "df[['a', 'b']].fillna(-1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. This one works\n",
    "df[['a', 'b']] = df[['a', 'b']].fillna(-1)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. This one also works ... strange, isn't it, after 1. ?\n",
    "df['c'].fillna(-1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of python code lines in PyAuDiSam project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Warning: Only python and markdown support for the moment\n",
    "def classifySourceLine(line, lang='python'):\n",
    "    empty, comment, code = 0, 0, 0\n",
    "    line = line.strip()\n",
    "    if not line:\n",
    "        empty = 1\n",
    "    elif line[0] == '#' or lang == 'markdown':\n",
    "        comment = 1\n",
    "    else:\n",
    "        code = 1\n",
    "    return empty, comment, code\n",
    "\n",
    "# Auto-tests\n",
    "assert classifySourceLine(' ') == (1, 0, 0)\n",
    "assert classifySourceLine(' #') == (0, 1, 0)\n",
    "assert classifySourceLine('x = 1') == (0, 0, 1)\n",
    "assert classifySourceLine('x = 1', lang='markdown') == (0, 1, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def countSourceLinesInFiles(files, lang='python', encoding='utf8'):\n",
    "    lsCounts = list()\n",
    "    for fpn in files:\n",
    "        print(fpn.as_posix())\n",
    "        sCounts = pd.Series(index=['nEmpties', 'nComments', 'nCode'], dtype=int, data=[0, 0, 0])\n",
    "        with open(fpn, encoding=encoding) as file:\n",
    "            if fpn.suffix == '.ipynb':  # Notebook file\n",
    "                ldCells = json.load(file)['cells']\n",
    "                for dCell in ldCells:\n",
    "                    for line in dCell['source']:\n",
    "                        sCounts += classifySourceLine(line, lang=dCell['cell_type'])\n",
    "            else:  # Normal source file\n",
    "                for line in file.readlines():\n",
    "                    sCounts += classifySourceLine(line, lang='python')\n",
    "        lsCounts.append(pd.Series(dict(file=fpn.name)).append(sCounts))\n",
    "    return pd.DataFrame(lsCounts)\n",
    "\n",
    "ExcludeFolderKeys = ['.git', 'venv/', '.ipynb_checkpoints', '__pycache__', 'build/', 'dist/']\n",
    "def countSourceLinesInFolder(folder, glob='*.py', recurse=False, lang='python'):\n",
    "    ldfCounts = list()\n",
    "    folder = pl.Path(folder)\n",
    "    if any(exclKey in folder.as_posix() for exclKey in ExcludeFolderKeys):\n",
    "        return None\n",
    "    dfCounts = countSourceLinesInFiles(folder.glob(glob), lang='python')\n",
    "    if not dfCounts.empty:\n",
    "        dfCounts.insert(0, 'folder', folder.as_posix())\n",
    "        ldfCounts.append(dfCounts)\n",
    "    if recurse:\n",
    "        for fileOrSubFolder in folder.iterdir():\n",
    "            if fileOrSubFolder.is_dir():\n",
    "                dfCounts = countSourceLinesInFolder(fileOrSubFolder, glob=glob, recurse=recurse, lang='python')\n",
    "                if not(dfCounts is None or dfCounts.empty):\n",
    "                    ldfCounts.append(dfCounts)\n",
    "    return None if not ldfCounts else pd.concat(ldfCounts)\n",
    "\n",
    "def countSourceLines(folder, glob='*.py', recurse=True, lang='python'):\n",
    "    dfCounts = countSourceLinesInFolder(folder, glob=glob, recurse=recurse, lang='python')\n",
    "    if dfCounts is None:\n",
    "        return None\n",
    "    dTotalHead = dict()\n",
    "    if recurse:\n",
    "        dTotalHead.update(folder='all')\n",
    "    dTotalHead.update(file='all')\n",
    "    dfCounts = dfCounts.append(pd.Series(dTotalHead).append(dfCounts[['nEmpties', 'nComments', 'nCode']].sum()),\n",
    "                               ignore_index=True)\n",
    "    return dfCounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In py sources (core and tests)\n",
    "countSourceLines('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countSourceLines('..', glob='*.ipynb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py38]",
   "language": "python",
   "name": "conda-env-py38-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
