{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!-- Auto table of contents -->\n",
    "<h1 class='tocIgnore'>Validation tests archives</h1>\n",
    "\n",
    "**pyaudisam**: Automation of Distance Sampling analyses with [Distance software](http://distancesampling.org/)\n",
    "\n",
    "Copyright (C) 2021 Jean-Philippe Meuret\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify it under the terms\n",
    "of the GNU General Public License as published by the Free Software Foundation,\n",
    "either version 3 of the License, or (at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n",
    "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n",
    "See the GNU General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU General Public License along with this program.\n",
    "If not, see https://www.gnu.org/licenses/.\n",
    "\n",
    "<p>For up-to-date validation tests, see <a href=\"./valtests.ipynb\" target=\"_blank\">valtests.ipynb</a></p>\n",
    "<div style=\"overflow-y: auto\">\n",
    "  <h2 class='tocIgnore'>Table of contents</h2>\n",
    "  <div id=\"toc\"></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "$.getScript('ipython_notebook_toc.js')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import pathlib as pl\n",
    "\n",
    "import re\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from IPython.display import HTML, Markdown\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#import plotly as ply\n",
    "#import plotly.graph_objs as plygo\n",
    "#import plotly.express as plyx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyaudisam as ads\n",
    "\n",
    "ads.runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create temporary directory if not yet done.\n",
    "tmpDir = pl.Path('tmp')\n",
    "tmpDir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logging configuration.\n",
    "ads.log.configure(handlers=[sys.stdout, tmpDir / 'valarc.log'], verbose=True, reset=True)\n",
    "\n",
    "ads.logger('matplotlib', level=ads.WARNING, reset=True)\n",
    "\n",
    "ads.logger('ads', level=ads.INFO, reset=True)\n",
    "#ads.logger('ads.eng', level=ads.INFO, reset=True)\n",
    "#ads.logger('ads.exr', level=ads.DEBUG, reset=True)\n",
    "#ads.logger('ads.dat', level=ads.DEBUG, reset=True)\n",
    "ads.logger('ads.rep', level=ads.INFO1, reset=True)\n",
    "#ads.logger('ads.opn', level=ads.DEBUG, reset=True)\n",
    "#ads.logger('ads.opr', level=ads.DEBUG, reset=True)\n",
    "#ads.logger('ads.anr', level=ads.DEBUG, reset=True)\n",
    "ads.logger('ads.onr', level=ads.DEBUG1, reset=True)\n",
    "\n",
    "logger = ads.logger('valarc', level=ads.DEBUG, reset=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Run analyses with real life field data (1/2 : long code, long run)\n",
    "\n",
    "Note: Don't use this low level method : MCDSAnalyser is here for than now.\n",
    "\n",
    "Here we use directly MCDSAnalysis class.\n",
    "\n",
    "(for comparison to manually issued analyses with Distance 7.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load analyses set specifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load refout results table\n",
    "refFileName = 'ACDC2019-Papyrus-ALAARV-TURMER-resultats-distance-73.xlsx'\n",
    "dfRefRes = pd.read_excel(pl.Path('refout', refFileName))\n",
    "dfRefRes.reset_index(inplace=True) # Generate analysis # (later need for original cases order)\n",
    "dfRefRes.rename(columns=dict(index='AnlysNum', Name='Model'), inplace=True)\n",
    "\n",
    "dfRefRes.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Build test cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate test cases definition code from refout results file (don't cheat : only input columns :-)\n",
    "modelIdCols = ['Model']\n",
    "modelParamCols = ['LTrunc', 'RTrunc', 'FitDistCuts', 'DiscrDistCuts']\n",
    "sampleSelCols = ['Species', 'Periods', 'Prec.', 'Duration']\n",
    "caseIdCols = ['AnlysNum', 'SampNum'] + sampleSelCols + modelIdCols\n",
    "\n",
    "dfRefRes['SampNum'] = dfRefRes.groupby(sampleSelCols, sort=False).ngroup()\n",
    "\n",
    "dfAnlysCases = dfRefRes[caseIdCols + modelParamCols].copy()\n",
    "\n",
    "dfAnlysCases['KeyFn'] = \\\n",
    "    dfAnlysCases.Model.apply(lambda s: 'UNIFORM' if s.startswith('Unif') \\\n",
    "                                                 else 'HNORMAL' if s.startswith('Half') else 'HAZARD')\n",
    "dfAnlysCases['AdjSer'] = \\\n",
    "    dfAnlysCases.Model.apply(lambda s: 'COSINE' if s.find(' Cos') > 0 \\\n",
    "                                                else 'POLY' if s.find(' SimPoly') > 0 else 'HERMITE')\n",
    "dfAnlysCases['InFileName'] = \\\n",
    "    dfAnlysCases.apply(lambda sRow: 'ACDC2019-Papyrus-{}-{}-{}mn-{}dec-dist.txt' \\\n",
    "                                    .format(sRow.Species,\n",
    "                                            'AB' if 'A+B' in sRow.Periods else 'A' if 'A' in sRow.Periods else 'B',\n",
    "                                            sRow.Duration.split(' ')[0], sRow['Prec.'].split(' ')[0]),\n",
    "                       axis='columns')\n",
    "dfAnlysCases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#def nan2None(v):\n",
    "#    return None if pd.isnull(v) else v\n",
    "def distCutsFromSpecs(v):\n",
    "    if pd.isnull(v):\n",
    "        return None\n",
    "    if isinstance(v, int):\n",
    "        return v\n",
    "    return [float(x) for x in v.split(',')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Prepare analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decimalFields = ['Point transect*Survey effort', 'Observation*Radial distance']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis engine (sequential)\n",
    "mcds = ads.MCDSEngine(workDir=tmpDir / 'mcds-out',\n",
    "                      executor=None, # Non-parallel: ~7.5s elapsed onWindows 10, 6-core i7-8850H, PCI-e SSD, \"optimal performances\" power scheme\n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Frozen analysis parameters (a choice here)\n",
    "KEstimCriterion = 'AIC'\n",
    "KCVInterval = 95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results object construction\n",
    "sampCols = [('sample', col, 'Value') for col in sampleSelCols]\n",
    "miSampCols = pd.MultiIndex.from_tuples(sampCols)\n",
    "\n",
    "sampIndCol = ('sample', 'SampNum', 'Value')\n",
    "custCols = [('sample', 'AnlysNum', 'Value'), sampIndCol] + sampCols + [('model', 'Model', 'Value')]\n",
    "miCustCols = pd.MultiIndex.from_tuples(custCols)\n",
    "\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols,\n",
    "                 data=dict(en=caseIdCols, fr=['NumAnlys', 'NumEchant', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Modèle']))\n",
    "\n",
    "results = ads.MCDSAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                     miSampleCols=miSampCols, sampleIndCol=sampIndCol,\n",
    "                                     distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                     surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3a. Or : Really run analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shorten test cases and reference results lists, to go faster\n",
    "# Warning: If you don't retain entire samples, later comparison will fail on Delta AIC values.\n",
    "#selCaseInds = [0, 5, 7, 22, 31] # Some random cases, with uncomplete samples.\n",
    "#selCaseInds = dfAnlysCases[dfAnlysCases.Sample.isin([3, 4])].index # A shorter selection, with complete samples.\n",
    "selCaseInds = range(len(dfAnlysCases)) # All of them.\n",
    "\n",
    "nOrigAnlysCases = len(dfAnlysCases)\n",
    "dfAnlysCases = dfAnlysCases.loc[selCaseInds]\n",
    "dfRefRes = dfRefRes.loc[selCaseInds]\n",
    "\n",
    "logger.info(f'Retained {len(selCaseInds)} out of {nOrigAnlysCases}.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Run all analyses\n",
    "lastInFileName = None\n",
    "for _, sCase in dfAnlysCases.iterrows():\n",
    "    \n",
    "    nCase = sCase.AnlysNum\n",
    "    name = sCase.InFileName[len('ACDC2019-Papyrus')+1:-len('-dist.txt')]\n",
    "    name += '-' + sCase.Model.lower().translate(str.maketrans({c:'-' for c in ' ,.:;()/'}))\n",
    "    logger.info(f'#{nCase+1:3d} {name} {sCase.KeyFn} {sCase.AdjSer}')\n",
    "    \n",
    "    # Create data set if not already done.\n",
    "    if lastInFileName != sCase.InFileName:\n",
    "        sds = ads.SampleDataSet(pl.Path('refin', sCase.InFileName), decimalFields=decimalFields)\n",
    "        lastInFileName = sCase.InFileName\n",
    "        \n",
    "    # Run analysis and get results\n",
    "    anlys = ads.MCDSAnalysis(engine=mcds, sampleDataSet=sds, name=name, logData=True,\n",
    "                             estimKeyFn=sCase.KeyFn, estimAdjustFn=sCase.AdjSer,\n",
    "                             estimCriterion=KEstimCriterion, cvInterval=KCVInterval,\n",
    "                             minDist=sCase.LTrunc, maxDist=sCase.RTrunc,\n",
    "                             fitDistCuts=distCutsFromSpecs(sCase.FitDistCuts),\n",
    "                             discrDistCuts=distCutsFromSpecs(sCase.DiscrDistCuts))\n",
    "\n",
    "    anlys.submit()\n",
    "    sResult = anlys.getResults()\n",
    "\n",
    "    # Save results\n",
    "    sHead = pd.Series(data=[sCase[col] for col in sCase.index[:len(caseIdCols)]], index=miCustCols)\n",
    "\n",
    "    results.append(sResult, sCustomHead=sHead)\n",
    "    \n",
    "# shutdown analysis engine\n",
    "mcds.shutdown()\n",
    "\n",
    "# Done.\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save results in case need for not recomputing them\n",
    "resFileName = os.path.join(mcds.workDir, 'valarc-results.xlsx')\n",
    "\n",
    "results.toExcel(resFileName, sheetName='adsvalarc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resFileName = os.path.join(mcds.workDir, 'valarc-results-en.xlsx')\n",
    "\n",
    "results.toExcel(resFileName, sheetName='Auto', lang='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check translation\n",
    "dfActTrRes = results.dfTransData('fr')\n",
    "\n",
    "dfActTrRes.head().T.iloc[:30] #.at['TroncGche', 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3b. Or : Load analyses from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#computed = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    resFileName = os.path.join(mcds.workDir, 'valarc-results.xlsx')\n",
    "    print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "    results.fromExcel(resFileName, sheetName='adsvalarc')\n",
    "    \n",
    "    # shutdown analysis engine\n",
    "    mcds.shutdown()\n",
    "\n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print('... {} analyses to compare'.format(len(results)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# II. Compare actual results to reference\n",
    "\n",
    "(reference = manually run analyses with Distance software)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Extract actual results to compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis results\n",
    "dfActRes = results.dfData\n",
    "\n",
    "dfActRes.head().T[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select columns of auto-results and match them with reference ones, for comparison.\n",
    "dCompCols = \\\n",
    "{\n",
    "    ('sample', 'AnlysNum', 'Value'):  'AnlysNum',\n",
    "    ('sample', 'SampNum', 'Value'):   'SampNum',\n",
    "    ('sample', 'Species', 'Value'):   'Species',\n",
    "    ('sample', 'Periods', 'Value'):   'Periods',\n",
    "    ('sample', 'Prec.', 'Value'):     'Prec.',\n",
    "    ('sample', 'Duration', 'Value'):  'Duration',\n",
    "    \n",
    "    ('model',  'Model', 'Value'):         'Model',\n",
    "    ('parameters', 'left truncation distance', 'Value'):           'LTrunc',\n",
    "    ('parameters', 'right truncation distance', 'Value'):          'RTrunc',\n",
    "    ('parameters', 'model fitting distance cut points', 'Value'):  'FitDistCuts',\n",
    "    ('parameters', 'distance discretisation cut points', 'Value'): 'DiscrDistCuts',\n",
    "    \n",
    "    ('run output', 'run status', 'Value'): 'Status',\n",
    "    #('run output', 'run time', 'Value'): 'Run', # Only for unintests ref. generation just below\n",
    "    \n",
    "    ('detection probability', 'total number of parameters (m)', 'Value'): '# params',\n",
    "    ('encounter rate', 'number of observations (n)', 'Value'): '# obs',\n",
    "    \n",
    "    #('detection probability', 'Delta AIC', 'Value'): 'Delta AIC',\n",
    "    ('detection probability', 'AIC value', 'Value'): 'AIC',\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value')               : 'GOF Chi-p',\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value')                  : 'GOF K-S p',\n",
    "    ('detection probability', 'Cramér-von Mises (uniform weighting) test probability', 'Value'): 'GOF CvM (unif) p',\n",
    "    ('detection probability', 'Cramér-von Mises (cosine weighting) test probability', 'Value') : 'GOF CvM (cos) p',\n",
    "    \n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'): 'ESW/EDR',\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Lcl')  : 'ESW/EDR LCL',\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Ucl')  : 'ESW/EDR UCL',\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Cv')   : 'ESW/EDR CV',\n",
    "    \n",
    "    ('density/abundance', 'density of animals', 'Value'): 'D',\n",
    "    ('density/abundance', 'density of animals', 'Lcl')  : 'D LCL',\n",
    "    ('density/abundance', 'density of animals', 'Ucl')  : 'D UCL',\n",
    "    ('density/abundance', 'density of animals', 'Cv')   : 'D CV',\n",
    "    \n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'): 'P',\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Lcl')  : 'P LCL',\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Ucl')  : 'P UCL',\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Cv')   : 'P CV',\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Df')   : 'P DF',\n",
    "}\n",
    "len(dCompCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Warning: Unused columns (full of NaNs) have been automatically removed\n",
    "# (see last line of AnalysisResultsSet.dfData getter)\n",
    "dCompCols = { k: v for k, v in dCompCols.items() if k in dfActRes.columns }\n",
    "len(dCompCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# So we need to cleanup modelParamCols too\n",
    "modelParamCols = [id_ for id_ in modelParamCols if id_ in dCompCols.values()]\n",
    "len(modelParamCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Safe stringification of model params\n",
    "# * needed for use in indexes (hashability)\n",
    "# * needed to cope with to_excel/read_excel inconsistent None management\n",
    "def modelParam2Str(par):\n",
    "    #print(par)\n",
    "    if isinstance(par, list):\n",
    "        spar = str([float(v) for v in par])\n",
    "    elif pd.isnull(par):\n",
    "        spar = 'None'\n",
    "    elif isinstance(par, str):\n",
    "        if ',' in par: # Assumed already somewhat stringified list\n",
    "            spar = str([float(v) for v in par.strip('[]').split(',')])\n",
    "    else:\n",
    "        spar = str(par)\n",
    "    return spar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select results columns and rename them as the reference is, for easier comparison\n",
    "dfActRes4c = dfActRes[list(dCompCols.keys())].copy()\n",
    "dfActRes4c.columns = [dCompCols[col] for col in dCompCols]\n",
    "dfActRes4c[modelParamCols] = dfActRes4c[modelParamCols].applymap(modelParam2Str) # Hashable mandatory for indexing\n",
    "dfActRes4c.set_index(caseIdCols + modelParamCols, inplace=True)\n",
    "\n",
    "dfActRes4c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select usefull reference columns for comparison\n",
    "dfRefRes4c = dfRefRes.copy()\n",
    "dfRefRes4c[modelParamCols] = dfRefRes4c[modelParamCols].applymap(modelParam2Str) # Hashable mandatory for indexing\n",
    "dfRefRes4c.set_index(caseIdCols + modelParamCols, inplace=True)\n",
    "dfRefRes4c = dfRefRes4c.reindex(columns=dfActRes4c.columns)\n",
    "\n",
    "dfRefRes4c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfActRes4c.to_excel('tmp/act-res.xlsx')\n",
    "#dfRefRes4c.to_excel('tmp/ref-res.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Automated diagnosis\n",
    "\n",
    "Note: Since then, ads.DataSet.compare has been developed based on this prototype ... use it !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First checks : equality of test case lists (index) and of column names (columns)\n",
    "assert sorted(dfActRes4c.index)   == sorted(dfRefRes4c.index)\n",
    "assert sorted(dfActRes4c.columns) == sorted(dfRefRes4c.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actual / reference closeness measure : -round(log10((actual - reference) / max(abs(actual), abs(reference))), 1)\n",
    "# = Compute the order of magnitude that separate the difference from the absolute max. of the two values\n",
    "# The greater it is, the lower the relative difference\n",
    "#    Ex: 3 = 10**3 ratio between difference absolue max. of the two,\n",
    "#        +inf = NO difference at all,\n",
    "#        0 = bad, one of the two is 0, and the other not,\n",
    "# See unitary test below.\n",
    "def closeness(sRefAct):\n",
    "    \n",
    "    x, y = sRefAct.to_list()\n",
    "    \n",
    "    # Special cases with 1 NaN, or 1 or more inf => all different\n",
    "    if np.isnan(x):\n",
    "        if not np.isnan(y):\n",
    "            return 0 # All different\n",
    "    elif np.isnan(y):\n",
    "        return 0 # All different\n",
    "    \n",
    "    if np.isinf(x) or np.isinf(y):\n",
    "        return 0 # All different\n",
    "    \n",
    "    # Normal case\n",
    "    c = abs(x - y)\n",
    "    if not np.isnan(c) and c != 0:\n",
    "        c = c / max(abs(x), abs(y))\n",
    "    \n",
    "    return np.inf if c == 0 else round(-np.log10(c), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actual / reference comparison : compute closeness indicator\n",
    "dfRelDif = dfRefRes4c.copy()\n",
    "for col in dfRelDif.columns:\n",
    "    dfRelDif['act'] = dfActRes4c[col]\n",
    "    dfRelDif[col] = dfRelDif[[col, 'act']].apply(closeness, axis='columns')\n",
    "    dfRelDif.drop(columns='act', inplace=True)\n",
    "    \n",
    "dfRelDif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Diagnosis : we only keep lines and columns with some relevant differences.\n",
    "dfBadRelDif = dfRelDif.copy()\n",
    "len(dfBadRelDif)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Suppress rows : Same status and NaNs in the remainder (if status == 0/3/4, execution error or no execution)\n",
    "valCols = [col for col in dfRelDif.columns if col != 'Status']\n",
    "dfBadRelDif.drop(dfBadRelDif[(dfBadRelDif.Status.abs() == np.inf) & dfBadRelDif[valCols].isnull().all(axis='columns')].index,\n",
    "                 axis='index', inplace=True)\n",
    "assert len(dfBadRelDif) == 29, len(dfBadRelDif)\n",
    "anlysNums = dfBadRelDif.index.get_level_values('AnlysNum').to_list()\n",
    "assert anlysNums == [0, 1, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14, 15, 17,\n",
    "                     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], anlysNums\n",
    "print(len(dfBadRelDif), 'analyses:', ', '.join(map(str, anlysNums)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Suppress rows : Status and all other columns == inf (<=> strict equality)\n",
    "#    NB. Some very small differences observed when results have just been computed or when they have been\n",
    "#        loaded from a previously saved Excel file (above 10**15 closeness value)\n",
    "dfBadRelDif.drop(dfBadRelDif[dfBadRelDif.apply(np.isinf, axis='columns').all(axis='columns')].index,\n",
    "                 axis='index', inplace=True)\n",
    "assert (computed and len(dfBadRelDif) <= 26) or (not computed and len(dfBadRelDif) <= 17), len(dfBadRelDif)\n",
    "anlysNums = dfBadRelDif.index.get_level_values('AnlysNum').to_list()\n",
    "assert (computed and all(anlysNum in [0, 1, 2, 5, 6, 7, 8, 9, 13, 14, 15, 17, 18,\n",
    "                                       19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]\n",
    "                         for anlysNum in anlysNums)) \\\n",
    "       or (not computed and all(anlysNum in [0, 1, 2, 7, 8, 9, 13, 14, 15, 19, 20, 23, 25, 27, 28, 29, 30]\n",
    "                                for anlysNum in anlysNums)), \\\n",
    "       anlysNums\n",
    "print(len(dfBadRelDif), 'analyses:', ', '.join(map(str, anlysNums)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Suppress rows : Status and all other columns >= à 15 (<=> nearly strict equality)\n",
    "dfBadRelDif.drop(dfBadRelDif[(dfBadRelDif >= 15).all(axis='columns')].index, axis='index', inplace=True)\n",
    "assert len(dfBadRelDif) == 5, len(dfBadRelDif)\n",
    "anlysNums = dfBadRelDif.index.get_level_values('AnlysNum').to_list()\n",
    "assert all(anlysNum in [9, 20, 28, 29, 30] for anlysNum in anlysNums), anlysNums\n",
    "print(len(dfBadRelDif), 'analyses:', ', '.join(map(str, anlysNums)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. Suppress rows : Same status and all other columns >= 4 (<=> close to equality)\n",
    "dfBadRelDif.drop(dfBadRelDif[(dfBadRelDif >= 4).all(axis='columns')].index, axis='index', inplace=True)\n",
    "assert len(dfBadRelDif) == 4, len(dfBadRelDif)\n",
    "anlysNums = dfBadRelDif.index.get_level_values('AnlysNum').to_list()\n",
    "assert all(anlysNum in [9, 20, 28, 30] for anlysNum in anlysNums), anlysNums\n",
    "print(len(dfBadRelDif), 'analyses:', ', '.join(map(str, anlysNums)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5. Suppress rows : Same status and all other columns >= 4 (<=> close to equality)\n",
    "#                    except for GOF KS and CvM, equal to NaN, because not computed when distances are discretised.\n",
    "if 'DiscrDistCuts' in dfBadRelDif.index.names:\n",
    "    discrCols = [col for col in dfRelDif.columns if not col.startswith('GOF') or col.find('Chi') > 0]\n",
    "    df2Drop = (dfBadRelDif.index.get_level_values('DiscrDistCuts') != -1) & (dfBadRelDif[discrCols] >= 4).all(axis='columns')\n",
    "    dfBadRelDif.drop(dfBadRelDif[df2Drop].index, axis='index', inplace=True)\n",
    "assert len(dfBadRelDif) == 2, len(dfBadRelDif)\n",
    "anlysNums = dfBadRelDif.index.get_level_values('AnlysNum').to_list()\n",
    "assert all(anlysNum in [9, 30] for anlysNum in anlysNums), anlysNums\n",
    "print(len(dfBadRelDif), 'analyses:', ', '.join(map(str, anlysNums)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Verdict (Cf. refFileName Excel file, sheet \"DiffAuto\" for explanations about the 2 different rows between Act/Ref)\n",
    "dfBadRelDif.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRefRes4c.loc[dfBadRelDif.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nFails = len(dfBadRelDif.index)\n",
    "if nFails > 0:\n",
    "    print(f'Warning: {nFails} test case(s) failed ;')\n",
    "    print(f' ... see sheet \"DiffAuto\" of {refFileName} for possible explanations.')\n",
    "else:\n",
    "    print('All test cases succeeded !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Save results to disk."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resCompFileName = os.path.join(mcds.workDir, 'valarc-rescomp.xlsx')\n",
    "\n",
    "with pd.ExcelWriter(resCompFileName) as xlsxWriter:\n",
    "\n",
    "    dfRefRes.to_excel(xlsxWriter, sheet_name='RefResults', index=True)\n",
    "    dfActRes4c.reset_index().to_excel(xlsxWriter, sheet_name='ActResults', index=False)\n",
    "    dfRelDif.reset_index().to_excel(xlsxWriter, sheet_name='Diff2Ref', index=False)\n",
    "    dfBadRelDif.reset_index().to_excel(xlsxWriter, sheet_name='BadDiff2Ref', index=False)\n",
    "    dfRefRes4c.loc[dfBadRelDif.index].reset_index().to_excel(xlsxWriter, sheet_name='RefResWithDiff', index=False)\n",
    "    dfActRes4c.loc[dfBadRelDif.index].reset_index().to_excel(xlsxWriter, sheet_name='ActResWithDiff', index=False)\n",
    "    dfActRes.to_excel(xlsxWriter, sheet_name='RawActResults', index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Build HTML and Excel reports\n",
    "\n",
    "See [IV. Excel and HTML reports](#IV.-Excel-and-HTML-reports)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# III. Parallel run of same analyses\n",
    "\n",
    "Note: Don't use this low level method : MCDSAnalyser is here for than now.\n",
    "\n",
    "Here, we directly call MCDSAnalysis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Prepare analyses\n",
    "\n",
    "(same test cases and input data as previously, for easy comparison)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis executor : 6, 8, None threads => min elapsed = ~2s on Windows 10, 6-core i7-8850H, PCI-e SSD, \"optimal performances\" power scheme\n",
    "parallelExecutor = ads.Executor(threads=6)\n",
    "\n",
    "# Analysis engine\n",
    "mcds = ads.MCDSEngine(workDir=tmpDir / 'mcds-pout', executor=parallelExecutor, \n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results object construction\n",
    "parResults = ads.MCDSAnalysisResultsSet(miCustomCols=miCustCols, miSampleCols=miSampCols, dfCustomColTrans=dfCustColTrans, \n",
    "                                        distanceUnit='Meter', areaUnit='Hectare', sampleIndCol=sampIndCol,\n",
    "                                        surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2a. Or : Really run analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shorten test cases and reference results lists, to go faster\n",
    "# Warning: If you don't retain entire samples, later comparison will fail on Delta AIC values.\n",
    "#selCaseInds = [0, 5, 7, 22, 31] # Some random cases, with uncomplete samples.\n",
    "#selCaseInds = dfAnlysCases[dfAnlysCases.Sample.isin([3, 4])].index # A shorter selection, with complete samples.\n",
    "selCaseInds = range(len(dfAnlysCases)) # All of them.\n",
    "\n",
    "nOrigAnlysCases = len(dfAnlysCases)\n",
    "dfAnlysCases = dfAnlysCases.loc[selCaseInds]\n",
    "dfRefRes = dfRefRes.loc[selCaseInds]\n",
    "\n",
    "logger.info(f'Retained {len(selCaseInds)} out of {nOrigAnlysCases}.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Start running all analyses\n",
    "lastInFileName = None\n",
    "analyses = dict()\n",
    "for _, sCase in dfAnlysCases.iterrows():\n",
    "    \n",
    "    nCase = sCase.AnlysNum\n",
    "    name = sCase.InFileName[len('ACDC2019-Papyrus')+1:-len('-dist.txt')]\n",
    "    name += '-' + sCase.Model.lower().translate(str.maketrans({c:'-' for c in ' ,.:;()/'}))\n",
    "    logger.info(f'#{nCase+1:3d} {name} {sCase.KeyFn} {sCase.AdjSer}')\n",
    "    \n",
    "    # Create data set if not already done.\n",
    "    if lastInFileName != sCase.InFileName:\n",
    "        sds = ads.SampleDataSet(pl.Path('refin', sCase.InFileName), decimalFields=decimalFields)\n",
    "        lastInFileName = sCase.InFileName\n",
    "        \n",
    "    # Start running analysis in parallel (don't wait for it's finished, go on)\n",
    "    sResHead = pd.Series(data=[sCase[col] for col in sCase.index[:len(caseIdCols)]], index=miCustCols)\n",
    "\n",
    "    anlys = ads.MCDSAnalysis(engine=mcds, sampleDataSet=sds, name=name, customData=sResHead, logData=True,\n",
    "                             estimKeyFn=sCase.KeyFn, estimAdjustFn=sCase.AdjSer,\n",
    "                             estimCriterion=KEstimCriterion, cvInterval=KCVInterval,\n",
    "                             minDist=sCase.LTrunc, maxDist=sCase.RTrunc,\n",
    "                             #minDist=nan2None(sCase.LTrunc), maxDist=nan2None(sCase.RTrunc),\n",
    "                             fitDistCuts=distCutsFromSpecs(sCase.FitDistCuts), # TODO: do this when building dfAnlysCases\n",
    "                             discrDistCuts=distCutsFromSpecs(sCase.DiscrDistCuts))\n",
    "    anlysFut = anlys.submit()\n",
    "    \n",
    "    # Store analysis object and associated \"future\" for later use (should be running soon or later).\n",
    "    analyses[anlysFut] = anlys\n",
    "    \n",
    "logger.info('All analyses started ; now waiting for their end, and results ...')\n",
    "\n",
    "# For each analysis as it gets completed (first completed => first yielded)\n",
    "for anlysFut in parallelExecutor.asCompleted(analyses):\n",
    "\n",
    "    # Retrieve analysis object from its associated future object\n",
    "    anlys = analyses[anlysFut]\n",
    "    \n",
    "    # Get analysis results\n",
    "    sResult = anlys.getResults()\n",
    "\n",
    "    # Save results with header\n",
    "    parResults.append(sResult, sCustomHead=anlys.customData)\n",
    "    \n",
    "# shutdown analysis engine\n",
    "mcds.shutdown()\n",
    "\n",
    "# Done.\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save results in case need for not recomputing them\n",
    "resFileName = os.path.join(mcds.workDir, 'valarc-results.xlsx')\n",
    "\n",
    "parResults.toExcel(resFileName, sheetName='adsvalarc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2b. Or : Load analyses from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    resFileName = os.path.join(mcds.workDir, 'valarc-results.xlsx')\n",
    "    print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "    parResults.fromExcel(resFileName, sheetName='adsvalarc')\n",
    "    \n",
    "    # shutdown analysis engine\n",
    "    mcds.shutdown()\n",
    "\n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print(f'... {len(parResults)} analyses to compare')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Compare parallel results to sequential ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare sequential results for comparison\n",
    "dfSeqCmpRes = results.dfTransData('en')\n",
    "\n",
    "dfSeqCmpRes.fillna(-9999, inplace=True) # Get rid of the Nan pb (because NaN != NaN :-)\n",
    "\n",
    "# Start date-time and elapsed time and folder can never be the same\n",
    "dfSeqCmpRes.drop(columns=['StartTime', 'ElapsedTime', 'RunFolder'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare parallel results for comparison\n",
    "dfParCmpRes = parResults.dfTransData('en')\n",
    "\n",
    "dfParCmpRes.sort_values(by='AnlysNum', inplace=True) # Back to original test case order = sequential run order\n",
    "\n",
    "dfParCmpRes.reset_index(inplace=True, drop=True) # Enforce same index as a consequence\n",
    "\n",
    "dfParCmpRes.fillna(-9999, inplace=True) # And get rid of the Nan pb (because NaN != Nan :-)\n",
    "\n",
    "# Start date-time and elapsed time and folder can never be the same\n",
    "dfParCmpRes.drop(columns=['StartTime', 'ElapsedTime', 'RunFolder'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Warning: Doesn't work if 1 of the 2 sets (not both) was loaded from disk (Excel numerical rounding stuff)\n",
    "assert (dfSeqCmpRes == dfParCmpRes).all().all(), \\\n",
    "       'Oh, oh, something went differently when run parallely ... but due to one results set loaded from disk ?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSeqCmpRes.compare(dfParCmpRes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IV. Excel and HTML reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this if you want to report parallel results.\n",
    "# Or NOT if you want to report sequential sequential\n",
    "seqResults = results\n",
    "results = parResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sélection des colonnes pour les tableaux de synthèse du rapport\n",
    "synthRepCols = \\\n",
    "[\n",
    "    ('sample', 'AnlysNum', 'Value'),\n",
    "    ('sample', 'Species', 'Value'),\n",
    "    ('sample', 'Periods', 'Value'),\n",
    "    ('sample', 'Prec.', 'Value'),\n",
    "    ('sample', 'Duration', 'Value'),\n",
    "    ('sample', 'SampNum', 'Value'),\n",
    "    ('sample stats', 'total number of observations', 'Value'),\n",
    "    ('sample stats', 'maximal observation distance', 'Value'),\n",
    "    \n",
    "    ('model', 'Model', 'Value'),\n",
    "    ('parameters', 'left truncation distance', 'Value'),\n",
    "    ('parameters', 'right truncation distance', 'Value'),\n",
    "    ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "    ('parameters', 'distance discretisation cut points', 'Value'),\n",
    "    \n",
    "    ('run output', 'run status', 'Value'),\n",
    "    \n",
    "    ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "    ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "    \n",
    "    ('detection probability', 'Delta AIC', 'Value'),\n",
    "    ('detection probability', 'AIC value', 'Value'),\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Delta Cv'),\n",
    "    ('density/abundance', 'density of animals', 'Cv'),\n",
    "    \n",
    "    ('encounter rate', 'observation rate', 'Value'),\n",
    "    ('combined quality', 'balanced 1', 'Value'),\n",
    "    ('combined quality', 'balanced 2', 'Value'),\n",
    "    ('combined quality', 'balanced 3', 'Value'),\n",
    "    ('combined quality', 'more Chi2', 'Value'),\n",
    "    ('combined quality', 'more KS', 'Value'),\n",
    "    ('combined quality', 'more DCv', 'Value'),\n",
    "    \n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Lcl'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Ucl'),\n",
    "    \n",
    "    ('density/abundance', 'density of animals', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Lcl'),\n",
    "    ('density/abundance', 'density of animals', 'Ucl'),\n",
    "    \n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Lcl'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Ucl'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Df'),\n",
    "\n",
    "    ('run output', 'run folder', 'Value'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select analysis results columns for the 3 textual columns of the synthesis pre-report\n",
    "sampleRepCols = \\\n",
    "[\n",
    "    ('sample', 'SampNum', 'Value'),\n",
    "    ('sample', 'Species', 'Value'),\n",
    "    ('sample', 'Periods', 'Value'),\n",
    "    ('sample', 'Prec.', 'Value'),\n",
    "    ('sample', 'Duration', 'Value'),\n",
    "    ('sample', 'AnlysNum', 'Value'),\n",
    "    ('sample stats', 'total number of observations', 'Value'),\n",
    "    ('sample stats', 'maximal observation distance', 'Value'),\n",
    "]\n",
    "\n",
    "paramRepCols = \\\n",
    "[\n",
    "    ('parameters', 'estimator key function', 'Value'),\n",
    "    ('parameters', 'estimator adjustment series', 'Value'),\n",
    "    ('parameters', 'left truncation distance', 'Value'),\n",
    "    ('parameters', 'right truncation distance', 'Value'),\n",
    "]\n",
    "    \n",
    "resultRepCols = \\\n",
    "[\n",
    "    ('run output', 'run status', 'Value'),\n",
    "    \n",
    "    ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "    ('encounter rate', 'left truncation distance', 'Value'),\n",
    "    ('encounter rate', 'right truncation distance (w)', 'Value'),\n",
    "    ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "    \n",
    "    ('detection probability', 'AIC value', 'Value'),\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "    ('combined quality', 'balanced 1', 'Value'),\n",
    "    ('combined quality', 'balanced 2', 'Value'),\n",
    "    ('combined quality', 'balanced 3', 'Value'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "\n",
    "    ('density/abundance', 'density of animals', 'Cv'),\n",
    "    ('density/abundance', 'density of animals', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Lcl'),\n",
    "    ('density/abundance', 'density of animals', 'Ucl'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortRepCols = \\\n",
    "[('sample', 'SampNum', 'Value'), ('sample', 'AnlysNum', 'Value')] \\\n",
    "+ [('sample', col, 'Value') for col in sampleSelCols] \\\n",
    "+ [('parameters', 'left truncation distance', 'Value'),\n",
    "   ('parameters', 'right truncation distance', 'Value'),\n",
    "   ('detection probability', 'Delta AIC', 'Value')]\n",
    "#   ('density/abundance', 'density of animals', 'Delta Cv')]\n",
    "\n",
    "sortRepAscend = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = ads.MCDSResultsFullReport(resultsSet=results, \n",
    "                                   sampleCols=sampleRepCols, paramCols=paramRepCols,\n",
    "                                   resultCols=resultRepCols, synthCols=synthRepCols,\n",
    "                                   sortCols=sortRepCols, sortAscend=sortRepAscend,\n",
    "                                   title='Validation PyAuDiSam : Analyses automatiques',\n",
    "                                   subTitle='Rapport d\\'analyse global', anlysSubTitle='Rapport détaillé',\n",
    "                                   description=(\"Résultats d'analyses exécutées \"\n",
    "                                                + ('en parallèle' if mcds.workDir.name.endswith('pout')\n",
    "                                                   else 'séquentiellement')),\n",
    "                                   keywords='pyaudisam, validation', pySources=['valarchives.ipynb'],\n",
    "                                   lang='fr', superSynthPlotsHeight=288,\n",
    "                                   #plotImgSize=(640, 400), plotLineWidth=1, plotDotWidth=4,\n",
    "                                   #plotFontSizes=dict(title=11, axes=10, ticks=9, legend=10),\n",
    "                                   tgtFolder=mcds.workDir, tgtPrefix='valarc-report')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlsxRep = report.toExcel()\n",
    "\n",
    "HTML(f'Rapport Excel : <a href=\"{xlsxRep}\" target=\"blank\">{xlsxRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme:\n",
    "# * 2021-02-13: 38s (n=3)\n",
    "htmlRep = report.toHtml(generators=6)\n",
    "\n",
    "HTML(f'Rapport HTML : <a href=\"{htmlRep}\" target=\"blank\">{htmlRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# V. Run and report pre-analyses (1/2 : long code, long duration)\n",
    "\n",
    "(to help users to setup the full analyses plan : run first try simple analyses and show PDF and few results)\n",
    "\n",
    "On same input data as for I, II, III.\n",
    "\n",
    "Note: Don't use this low level method : MCDSPreAnalyser is here for than now.\n",
    "\n",
    "Here we directly call MCDSPreAnalysis class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Determine samples from input data\n",
    "\n",
    "* in real life, we'd simply load field collected data, and deduce individual \"samples\" from it ;\n",
    "* but there, for testing, it's easier to deduce samples from manual analysis specification file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create sample table from refout results table\n",
    "refFileName = 'ACDC2019-Papyrus-ALAARV-TURMER-resultats-distance-73.xlsx'\n",
    "\n",
    "sampleSelCols = ['Species', 'Periods', 'Prec.', 'Duration']\n",
    "\n",
    "dfSamples = pd.read_excel(pl.Path('refout', refFileName), usecols=sampleSelCols)\n",
    "dfSamples.rename(columns=dict(Name='Model'), inplace=True)\n",
    "dfSamples.drop_duplicates(inplace=True)\n",
    "dfSamples.reset_index(drop=True, inplace=True)\n",
    "\n",
    "dfSamples.reset_index(inplace=True) # Generate sample # (later need for original sample order)\n",
    "sampleIndCol = 'SampleNum'\n",
    "dfSamples.rename(columns=dict(index=sampleIndCol), inplace=True)\n",
    "\n",
    "sampleSelCols = [sampleIndCol] + sampleSelCols\n",
    "\n",
    "dfSamples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Prepare pre-analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decimalFields = ['Point transect*Survey effort', 'Observation*Radial distance']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis engine: Non-parallel executor here, 'cause MCDSPreAnalysis takes care of this !\n",
    "mcds = ads.MCDSEngine(workDir=tmpDir / 'mcds-preout', \n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results object construction\n",
    "custCols = [('sample', col, 'Value') for col in sampleSelCols]\n",
    "sampMIndCol = next(iter(mCol for mCol in custCols if mCol[1] == sampleIndCol))\n",
    "miCustCols = pd.MultiIndex.from_tuples(custCols)\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols,\n",
    "                 data=dict(en=sampleSelCols, fr=['NumEchant', 'Espèce', 'Périodes', 'Préc.', 'Durée']))\n",
    "\n",
    "preResults = ads.MCDSPreAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans, sampleIndCol=sampMIndCol,\n",
    "                                           distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                           surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KPreEstimCrit = 'AIC'\n",
    "KPreCVInterval = 95\n",
    "KPreEstimModStrat = [dict(keyFn=kf, adjSr='COSINE', estCrit=KPreEstimCrit, cvInt=KPreCVInterval) \\\n",
    "                     for kf in['HNORMAL', 'HAZARD', 'UNIFORM', 'NEXPON']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 Or : Really run pre-analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Run all analyses\n",
    "lastInFileName = None\n",
    "for _, sSamp in dfSamples.iterrows():\n",
    "    \n",
    "    nSamp = sSamp.SampleNum\n",
    "    sampId = '{}-{}-{}mn-{}dec' \\\n",
    "             .format(sSamp.Species,\n",
    "                     'AB' if 'A+B' in sSamp.Periods else 'A' if 'A' in sSamp.Periods else 'B',\n",
    "                     sSamp.Duration.split(' ')[0], sSamp['Prec.'].split(' ')[0])\n",
    "    logger.info(f'#{nSamp+1:3d} {sampId}')\n",
    "    \n",
    "    # Create data set if not already done.\n",
    "    inFileName = 'ACDC2019-Papyrus-{}-dist.txt'.format(sampId)\n",
    "    if lastInFileName != inFileName:\n",
    "        sds = ads.SampleDataSet(pl.Path('refin', inFileName), decimalFields=decimalFields)\n",
    "        lastInFileName = inFileName\n",
    "        \n",
    "    # Run analysis: Not parallel runs for now ... see below.\n",
    "    preAnlys = ads.MCDSPreAnalysis(engine=mcds, sampleDataSet=sds, name=sampId, executor=None,\n",
    "                                   logData=False, modelStrategy=KPreEstimModStrat)\n",
    "    preAnlys.submit()\n",
    "    \n",
    "    # Get results (wait for it's finished)\n",
    "    sResult = preAnlys.getResults()\n",
    "\n",
    "    # Save results\n",
    "    sResHead = sSamp.copy()\n",
    "    sResHead.index = miCustCols\n",
    "    preResults.append(sResult, sCustomHead=sResHead)\n",
    "    \n",
    "# shutdown analysis engine\n",
    "mcds.shutdown()\n",
    "\n",
    "# Done.\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at results\n",
    "preResults.dfTransData('fr')[['NumEchant', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Fn Clé',\n",
    "                              'Sér Ajust', 'CodEx', 'NObs', 'AIC', 'Chi2 P', 'KS P', \n",
    "                              'Densité', 'CoefVar Densité', 'Min Densité', 'Max Densité']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save results in case need for not recomputing them\n",
    "resFileName = os.path.join(mcds.workDir, 'valarc-preresults.xlsx')\n",
    "\n",
    "preResults.toExcel(resFileName, sheetName='adsvalarc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Or : Load pre-analyses results from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    resFileName = os.path.join(mcds.workDir, 'valarc-preresults.xlsx')\n",
    "    print(f'Loading pre-results from {resFileName} ...')\n",
    "\n",
    "    preResults.fromExcel(resFileName, sheetName='adsvalarc')\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print(f'... {len(preResults)} pre-analyses loaded')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at results\n",
    "preResults.dfTransData('fr')[['NumEchant', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Fn Clé',\n",
    "                              'Sér Ajust', 'CodEx', 'NObs', 'AIC', 'Chi2 P', 'KS P', \n",
    "                              'Densité', 'CoefVar Densité', 'Min Densité', 'Max Densité']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. HTML and Excel pre-analyses reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this if you want to report parallel results.\n",
    "# Or NOT if you want to report sequential sequential\n",
    "seqPreResults = preResults\n",
    "preResults = parPreResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sélection des colonnes pour les tableaux de synthèse du rapport\n",
    "synthPreRepCols = \\\n",
    "[\n",
    "    ('sample', 'SampleNum', 'Value'),\n",
    "    ('sample', 'Species', 'Value'),\n",
    "    ('sample', 'Periods', 'Value'),\n",
    "    ('sample', 'Prec.', 'Value'),\n",
    "    ('sample', 'Duration', 'Value'),\n",
    "    ('sample stats', 'total number of observations', 'Value'),\n",
    "    ('sample stats', 'maximal observation distance', 'Value'),\n",
    "    \n",
    "    ('parameters', 'estimator key function', 'Value'),\n",
    "    ('parameters', 'estimator adjustment series', 'Value'),\n",
    "    ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "    ('parameters', 'distance discretisation cut points', 'Value'),\n",
    "    \n",
    "    ('run output', 'run status', 'Value'),\n",
    "    \n",
    "    ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "    ('encounter rate', 'right truncation distance (w)', 'Value'),\n",
    "    ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "    \n",
    "    ('detection probability', 'AIC value', 'Value'),\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Cv'),\n",
    "    ('encounter rate', 'observation rate', 'Value'),\n",
    "    ('combined quality', 'balanced 1', 'Value'),\n",
    "    ('combined quality', 'balanced 2', 'Value'),\n",
    "    ('combined quality', 'balanced 3', 'Value'),\n",
    "    ('combined quality', 'more Chi2', 'Value'),\n",
    "    ('combined quality', 'more KS', 'Value'),\n",
    "    ('combined quality', 'more DCv', 'Value'),\n",
    "    \n",
    "    \n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Lcl'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Ucl'),\n",
    "    \n",
    "    ('density/abundance', 'density of animals', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Lcl'),\n",
    "    ('density/abundance', 'density of animals', 'Ucl'),\n",
    "    \n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Lcl'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Ucl'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Df'),\n",
    "\n",
    "    ('run output', 'run folder', 'Value'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select analysis results columns for the 3 textual columns of the synthesis pre-report\n",
    "samplePreRepCols = \\\n",
    "[\n",
    "    ('sample', 'SampleNum', 'Value'),\n",
    "    ('sample', 'Species', 'Value'),\n",
    "    ('sample', 'Periods', 'Value'),\n",
    "    ('sample', 'Prec.', 'Value'),\n",
    "    ('sample', 'Duration', 'Value'),\n",
    "    ('sample stats', 'total number of observations', 'Value'),\n",
    "    ('sample stats', 'maximal observation distance', 'Value'),\n",
    "]\n",
    "\n",
    "paramPreRepCols = \\\n",
    "[\n",
    "    ('parameters', 'estimator key function', 'Value'),\n",
    "    ('parameters', 'estimator adjustment series', 'Value'),\n",
    "    ('parameters', 'CV interval', 'Value')\n",
    "]\n",
    "    \n",
    "resultPreRepCols = \\\n",
    "[\n",
    "    ('run output', 'run status', 'Value'),\n",
    "    \n",
    "    ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "    ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "    \n",
    "    ('detection probability', 'AIC value', 'Value'),\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "\n",
    "    ('encounter rate', 'observation rate', 'Value'),\n",
    "    ('combined quality', 'balanced 1', 'Value'),\n",
    "    ('combined quality', 'balanced 2', 'Value'),\n",
    "    ('combined quality', 'balanced 3', 'Value'),\n",
    "    ('combined quality', 'more Chi2', 'Value'),\n",
    "    ('combined quality', 'more KS', 'Value'),\n",
    "    ('combined quality', 'more DCv', 'Value'),\n",
    "    \n",
    "    ('density/abundance', 'density of animals', 'Cv'),\n",
    "    ('density/abundance', 'density of animals', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Lcl'),\n",
    "    ('density/abundance', 'density of animals', 'Ucl'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortPreRepCols = [('sample', 'SampleNum', 'Value')]\n",
    "\n",
    "sortPreRepAscend = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preReport = ads.MCDSResultsPreReport(resultsSet=preResults,\n",
    "                                     title='Validation PyAuDiSam : Pré-analyses', subTitle='Rapport de pré-analyse',\n",
    "                                     anlysSubTitle='Détail des pré-analyses',\n",
    "                                     description=(\"Résultats de pré-analyses exécutées \"\n",
    "                                                  + ('en parallèle' if mcds.workDir.name.endswith('pout')\n",
    "                                                     else 'séquentiellement')),\n",
    "                                     keywords='pyaudisam, validation', lang='fr', superSynthPlotsHeight=288,\n",
    "                                     #plotImgSize=(640, 400), plotLineWidth=1, plotDotWidth=4,\n",
    "                                     #plotFontSizes=dict(title=11, axes=10, ticks=9, legend=10),\n",
    "                                     sampleCols=samplePreRepCols, paramCols=paramPreRepCols,\n",
    "                                     resultCols=resultPreRepCols, synthCols=synthPreRepCols,\n",
    "                                     sortCols=sortPreRepCols, sortAscend=sortPreRepAscend,\n",
    "                                     tgtFolder=mcds.workDir, tgtPrefix='valarc-prereport')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlsxPreRep = preReport.toExcel()\n",
    "\n",
    "HTML(f'Rapport Excel : <a href=\"{xlsxPreRep}\" target=\"blank\">{xlsxPreRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "htmlPreRep = preReport.toHtml()\n",
    "\n",
    "HTML(f'Pré-rapport HTML : <a href=\"{htmlPreRep}\" target=\"blank\">{htmlPreRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preResults.specs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VI. Parallel run of same pre-analyses (2/2 : long code, short duration)\n",
    "\n",
    "And compare results to sequential run's.\n",
    "\n",
    "Note: Don't use this low level method : MCDSPreAnalyser is here for than now.\n",
    "\n",
    "Here we directly call MCDSPreAnalysis class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Prepare analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis engine : Non-parallel executor here: MCDSPreAnalysis takes care of this !\n",
    "mcds = ads.MCDSEngine(workDir=tmpDir / 'mcds-prepout', \n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results object construction\n",
    "parPreResults = ads.MCDSPreAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                              sampleIndCol=sampMIndCol, \n",
    "                                              distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                              surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 Or : Really run pre-analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pre-analysis executor (kind of overkill here, with only 5 pre-analyses ... but still works twice as rapidly !).\n",
    "parallelExecutor = ads.Executor(threads=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Run all analyses\n",
    "lastInFileName = None\n",
    "parPreAnalyses = dict()\n",
    "for _, sSamp in dfSamples.iterrows():\n",
    "    \n",
    "    nSamp = sSamp.SampleNum\n",
    "    sampId = '{}-{}-{}mn-{}dec' \\\n",
    "             .format(sSamp.Species,\n",
    "                     'AB' if 'A+B' in sSamp.Periods else 'A' if 'A' in sSamp.Periods else 'B',\n",
    "                     sSamp.Duration.split(' ')[0], sSamp['Prec.'].split(' ')[0])\n",
    "    logger.info(f'#{nSamp+1:3d} {sampId}')\n",
    "    \n",
    "    # Create data set if not already done.\n",
    "    inFileName = 'ACDC2019-Papyrus-{}-dist.txt'.format(sampId)\n",
    "    if lastInFileName != inFileName:\n",
    "        sds = ads.SampleDataSet(pl.Path('refin', inFileName), decimalFields=decimalFields)\n",
    "        lastInFileName = inFileName\n",
    "        \n",
    "    # Start running analysis (but don't wait for it's finished)\n",
    "    sResHead = sSamp.copy()\n",
    "    sResHead.index = miCustCols\n",
    "    \n",
    "    preAnlys = ads.MCDSPreAnalysis(engine=mcds, sampleDataSet=sds, name=sampId,\n",
    "                                   customData=sResHead, executor=parallelExecutor,\n",
    "                                   logData=False, modelStrategy=KPreEstimModStrat)\n",
    "    preAnlysFut = preAnlys.submit()\n",
    "    \n",
    "    # Store analysis object and associated \"future\" for later use (should be running soon or later).\n",
    "    parPreAnalyses[preAnlysFut] = preAnlys\n",
    "    \n",
    "logger.info('All pre-analyses started ; now waiting for their end, and results ...')\n",
    "\n",
    "# For each analysis as it gets completed (first completed => first yielded)\n",
    "for preAnlysFut in parallelExecutor.asCompleted(parPreAnalyses):\n",
    "\n",
    "    # Retrieve pre-analysis object from its associated future object\n",
    "    preAnlys = parPreAnalyses[preAnlysFut]\n",
    "    \n",
    "    # Get pre-analysis results\n",
    "    sResult = preAnlys.getResults()\n",
    "\n",
    "    # Save results with header\n",
    "    parPreResults.append(sResult, sCustomHead=preAnlys.customData)\n",
    "    \n",
    "# shutdown executor\n",
    "parallelExecutor.shutdown()\n",
    "\n",
    "# shutdown analysis engine\n",
    "mcds.shutdown()\n",
    "\n",
    "# Done.\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at results\n",
    "parPreResults.dfTransData('fr')[['NumEchant', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Fn Clé',\n",
    "                                 'Sér Ajust', 'CodEx', 'NObs', 'AIC', 'Chi2 P', 'KS P', \n",
    "                                 'Densité', 'CoefVar Densité', 'Min Densité', 'Max Densité']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save results in case need for not recomputing them\n",
    "resFileName = os.path.join(mcds.workDir, 'valarc-preresults-par.xlsx')\n",
    "\n",
    "parPreResults.toExcel(resFileName, sheetName='adsvalarc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Or : Load analyses from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    resFileName = os.path.join(mcds.workDir, 'valarc-preresults-par.xlsx')\n",
    "    print('Loading pre-results from {} ...'.format(resFileName))\n",
    "\n",
    "    parPreResults.fromExcel(resFileName, sheetName='adsvalarc')\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print('... {} pre-analyses loaded'.format(len(parPreResults)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look at results\n",
    "parPreResults.dfTransData('fr')[['NumEchant', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Fn Clé',\n",
    "                                 'Sér Ajust', 'CodEx', 'NObs', 'AIC', 'Chi2 P', 'KS P', \n",
    "                                 'Densité', 'CoefVar Densité', 'Min Densité', 'Max Densité']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Compare parallel results to sequential ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare sequential results for comparison\n",
    "dfSeqCmpRes = preResults.dfTransData('en')\n",
    "\n",
    "dfSeqCmpRes.fillna(-9999, inplace=True) # Get rid of the Nan pb (because NaN != NaN :-)\n",
    "\n",
    "# Start date-time and elapsed time and folder can never be the same\n",
    "dfSeqCmpRes.drop(columns=['StartTime', 'ElapsedTime', 'RunFolder'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare parallel results for comparison\n",
    "dfParCmpRes = parPreResults.dfTransData('en')\n",
    "\n",
    "dfParCmpRes.sort_values(by='SampleNum', inplace=True) # Back to original test case order = sequential run order\n",
    "\n",
    "dfParCmpRes.reset_index(inplace=True, drop=True) # Enforce same index as a consequence\n",
    "\n",
    "dfParCmpRes.fillna(-9999, inplace=True) # And get rid of the Nan pb (because NaN != Nan :-)\n",
    "\n",
    "# Start date-time and elapsed time and folder can never be the same\n",
    "dfParCmpRes.drop(columns=['StartTime', 'ElapsedTime', 'RunFolder'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (dfSeqCmpRes == dfParCmpRes).all().all(), 'Oh, oh, something went differently when run parallely ...'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Build HTML and Excel pre-analyses reports\n",
    "\n",
    "See [4. HTML and Excel pre-analyses reports](#4.-HTML-and-Excel-pre-analyses-reports)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VII. Truncation optimisation (short code and fast run)\n",
    "\n",
    "Thanks to MCDSZeroOrderTruncationOptimiser class.\n",
    "\n",
    "Note: Don't use this low level method, MCDSTruncationOptanalyser class is now here for that (easier from far, and shorter code)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    ads.logger('ads.opr', level=ads.DEBUG, reset=True)\n",
    "    ads.logger('ads.dat', level=ads.DEBUG, reset=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimAbbrev(sAnlys):\n",
    "    \n",
    "    # Sample abbreviation\n",
    "    spcAbbrev = ''.join(word[:4].title() for word in sAnlys['Espèce'].split(' ')[:2])\n",
    "    sampAbbrev = [str(x) for x in [spcAbbrev, sAnlys.Passage.replace('+', ''),\n",
    "                                   sAnlys.Adulte.replace('+', ''), sAnlys['Durée']]]\n",
    "\n",
    "    # Model + Parameters abbreviation\n",
    "    modParAbbrev = [sAnlys['FonctionClé'][:3].lower(), sAnlys['SérieAjust'][:3].lower()]\n",
    "    \n",
    "    return '-'.join(sampAbbrev + modParAbbrev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. Optimiser parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source / Results data\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'Distance'\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'NumEchant'\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "\n",
    "sampleAbbrevCol = 'AbrevEchant'\n",
    "\n",
    "optIndCol = 'IndOptim'\n",
    "optAbbrevCol = 'AbrevOptim'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Les paramètres généraux d'analyse DS\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Hectare'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "clustering = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Default optimisation params.\n",
    "defEstimKeyFn = 'HNORMAL'\n",
    "defEstimAdjustFn = 'COSINE'\n",
    "defEstimCriterion = 'AIC'\n",
    "defCVInterval = 95\n",
    "defMinDist = None\n",
    "defMaxDist = None, \n",
    "defFitDistCuts = None\n",
    "defDiscrDistCuts = None\n",
    "\n",
    "defExpr2Optimise = 'chi2'\n",
    "defMinimiseExpr = False\n",
    "defOutliersMethod = 'tucquant'\n",
    "defOutliersQuantCutPct = 7\n",
    "defFitDistCutsFctr = ads.Interval(min=0.6, max=1.4)\n",
    "defDiscrDistCutsFctr = ads.Interval(min=0.5, max=1.2)\n",
    "\n",
    "defSubmitTimes = 1\n",
    "defSubmitOnlyBest = None\n",
    "\n",
    "defCoreEngine = 'zoopt'\n",
    "defCoreMaxIters = 100\n",
    "defCoreTermExprValue = None\n",
    "defCoreAlgorithm = 'racos'\n",
    "defCoreMaxRetries = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Individuals data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv = ads.DataSet('refin/ACDC2019-Naturalist-ExtraitObsIndiv.ods', sheet='DonnéesIndiv').dfData\n",
    "dfObsIndiv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{ col: dfObsIndiv[col].unique() for col in ['Observateur', 'Point', 'Passage', 'Adulte', 'Durée', 'Espèce'] }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Actual transects\n",
    "\n",
    "(can't deduce them from data, some points are missing because of data selection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTransects = ads.DataSet('refin/ACDC2019-Naturalist-ExtraitObsIndiv.ods', sheet='Inventaires').dfData\n",
    "dfTransects"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Samples and analyses to optimise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "workDir = tmpDir / 'mcds-optr'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. For testing all optimisation parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "varOpt = '-all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRawOptimExplSpecs = ads.Analyser.explicitVariantSpecs('refin/ACDC2019-Naturalist-ExtraitSpecsAnalyses.xlsx', \n",
    "                                                        ignore=['Params3_expl'])\n",
    "\n",
    "# No use of these cols, as we'll compute them !\n",
    "dfRawOptimExplSpecs = dfRawOptimExplSpecs.drop(columns=['TrGche', 'TrDrte', 'NbTrchMod']) \\\n",
    "                                         .drop_duplicates().reset_index(drop=True)\n",
    "\n",
    "nOptimExplSpecs = len(dfRawOptimExplSpecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add optim. params\n",
    "dfMoreOptimCols = pd.DataFrame([dict(CritChx='AIC', IntervConf=95,\n",
    "                                     TrGche='auto', TrDrte='auto', MethOutliers='tucquant(2.5)',\n",
    "                                     NbTrchMod='mult(2/3, 3/2)', NbTrDiscr=None,\n",
    "                                     #TroncGche='auto', TroncDrte='auto', MethOutliers='tucquant(2.5)',\n",
    "                                     #NbTrModel='mult(2/3, 3/2)', NbTrDiscr=None,\n",
    "                                     ExprOpt='max(chi2)', MoteurOpt='zoopt(160)')]*len(dfRawOptimExplSpecs))\n",
    "\n",
    "dfRawOptimExplSpecs = pd.concat([dfRawOptimExplSpecs, dfMoreOptimCols], axis='columns')\n",
    "dfRawOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nOptimedAnlyses = nOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes de dfRawOptimExplSpecs donnant les paramètres d'analyse / optimisation\n",
    "optimParamSpecCols  = ['FonctionClé', 'SérieAjust', 'CritChx', 'IntervConf',\n",
    "                       'TrGche', 'TrDrte', 'MethOutliers', 'NbTrchMod', 'NbTrDiscr',\n",
    "                       #'TroncGche', 'TroncDrte', 'MethOutliers', 'NbTrModel', 'NbTrDiscr',\n",
    "                       'ExprOpt', 'MoteurOpt']\n",
    "\n",
    "# Et en version interne\n",
    "intOptimParamSpecCols = ['EstimKeyFn', 'EstimAdjustFn', 'EstimCriterion', 'CvInterval',\n",
    "                          'MinDist', 'MaxDist', 'OutliersMethod', 'FitDistCuts', 'DiscrDistCuts',\n",
    "                          'Expr2Optimise', 'OptimisationCore']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Or: Only main optimisation parameters\n",
    "\n",
    "* for comparison with X below,\n",
    "* for comparing results goodness with various optimisation parameters, in XI below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "varOpt = '-main'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRawOptimExplSpecs = ads.Analyser.explicitVariantSpecs('refin/ACDC2019-Naturalist-ExtraitSpecsOptanalyses.xlsx', \n",
    "                                                        ignore=['Params1_expl', 'Params2_expl'])\n",
    "\n",
    "dfRawOptimExplSpecs.drop(dfRawOptimExplSpecs[dfRawOptimExplSpecs[['TrGche', 'TrDrte', 'NbTrchMod', 'MultiOpt']]\n",
    "                                    .isnull().all(axis='columns')].index,\n",
    "                         inplace=True)\n",
    "\n",
    "dfRawOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nOptimExplSpecs = len(dfRawOptimExplSpecs) - dfRawOptimExplSpecs.duplicated().sum()  # Duplicates will be removed\n",
    "\n",
    "nOptimedAnlyses = 22  # See MultiOpt col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes de dfRawOptimExplSpecs donnant les paramètres d'analyse / optimisation\n",
    "optimParamSpecCols  = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod', 'MultiOpt']\n",
    "\n",
    "# Et en version interne\n",
    "intOptimParamSpecCols = ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts', 'SubmitParams']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4A. Or : Really run optimisations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. MCDS Zeroth Order Truncation Optimiser object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zoptr = ads.MCDSZerothOrderTruncationOptimiser \\\n",
    "                (dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                 transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                 sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                 abbrevCol=optAbbrevCol, abbrevBuilder=optimAbbrev,\n",
    "                 anlysIndCol=optIndCol, sampleIndCol=sampleNumCol,\n",
    "                 distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                 surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                 resultsHeadCols=dict(before=[optIndCol], sample=sampleSelCols, after=optimParamSpecCols),\n",
    "                 workDir=workDir, runMethod='os.system', runTimeOut=None, logProgressEvery=1, backupEvery=5,\n",
    "                 defEstimKeyFn=defEstimKeyFn, defEstimAdjustFn=defEstimAdjustFn,\n",
    "                 defEstimCriterion=defEstimCriterion, defCVInterval=defCVInterval,\n",
    "                 defExpr2Optimise=defExpr2Optimise, defMinimiseExpr=defMinimiseExpr,\n",
    "                 defOutliersMethod=defOutliersMethod, defOutliersQuantCutPct=defOutliersQuantCutPct,\n",
    "                 defFitDistCutsFctr=defFitDistCutsFctr, defDiscrDistCutsFctr=defDiscrDistCutsFctr,\n",
    "                 defSubmitTimes=defSubmitTimes, defSubmitOnlyBest=defSubmitOnlyBest,\n",
    "                 defCoreMaxIters=defCoreMaxIters, defCoreTermExprValue=defCoreTermExprValue,\n",
    "                 defCoreAlgorithm=defCoreAlgorithm, defCoreMaxRetries=defCoreMaxRetries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Run optimisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOptimExplSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, checkVerdict, checkErrors = \\\n",
    "    zoptr.explicitParamSpecs(dfExplParamSpecs=dfRawOptimExplSpecs, dropDupes=True, check=True)\n",
    "\n",
    "print(dict(checkVerdict=checkVerdict, checkErrors=checkErrors, \n",
    "           nActualOptimExplSpecs=len(dfOptimExplSpecs), nExpectedOptimExplSpecs=nOptimExplSpecs,\n",
    "           optimParamSpecCols=optimParamSpecCols, intOptimParamSpecCols=intOptimParamSpecCols,\n",
    "           unmUserParamSpecCols=unmUserParamSpecCols))\n",
    "\n",
    "assert len(dfOptimExplSpecs) == nOptimExplSpecs\n",
    "assert userParamSpecCols == optimParamSpecCols\n",
    "assert intParamSpecCols == intOptimParamSpecCols\n",
    "assert unmUserParamSpecCols == []\n",
    "assert checkVerdict\n",
    "assert not checkErrors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Analyses\n",
    "results = zoptr.run(dfOptimExplSpecs, threads=12)\n",
    "\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Target:**\n",
    "* Computer: Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme\n",
    "* runMethod: os.system\n",
    "\n",
    "**Variant 3a: \"all\"**\n",
    "\n",
    "* 22 analyses specs (12 parallel threads)\n",
    "* Paramètres dans refin/ACDC2019-Naturalist-ExtraitSpecsOptanalyses.xlsx nettoyé de Param1_expl et Param2_expl.\n",
    "* MoteurOpt='zoopt(120)' => 2020-08-21 22:41:56,626 2880 analyses => 22 results, Wall time: 4min 21s\n",
    "* MoteurOpt='zoopt(160)' => 2021-08-22 1920 analyses => 12 results, Wall time: 2min 44s\n",
    "\n",
    "**Variant 3b: \"main\"**\n",
    "\n",
    "* 12 analyses specs (6 parallel threads)\n",
    "* CritChx='AIC', IntervConf=95, TroncGche='auto', TroncDrte='auto', MethOutliers='tucquant(2.5)', NbTrModel='mult(2/3, 3/2)', NbTrDiscr=None, ExprOpt='max(chi2)'\n",
    "* MoteurOpt='zoopt(160)' => 2020-06-29 21:18:04,727 Wall time: 4min 31s\n",
    "* MoteurOpt='zoopt(250, tv=0.6)' => 2020-06-28 19:23:38,868 Wall time: 9min 19s\n",
    "\n",
    "**Variant 3b: \"main\"**\n",
    "\n",
    "* 12 analyses specs (12 parallel threads)\n",
    "* CritChx='AIC', IntervConf=95, TroncGche='auto', TroncDrte='auto', MethOutliers='tucquant(2.5)', NbTrModel='mult(2/3, 3/2)', NbTrDiscr=None, ExprOpt='max(chi2)'\n",
    "* MoteurOpt='zoopt(160)' => 2020-07-18 15:22:47,289 Wall time: 3min 51s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Target:**\n",
    "* Computer: Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme\n",
    "* runMethod: subprocess.run\n",
    "\n",
    "**Variant 3a: \"all\"**\n",
    "\n",
    "* 12 analyses specs (12 parallel threads)\n",
    "* Parameters from refin/ACDC2019-Naturalist-ExtraitSpecsOptanalyses.xlsx, after removing Param1_expl and Param2_expl.\n",
    "* Default: CritChx='AIC', IntervConf=95, MethOutliers='tucquant(7)', NbTrModel='mult(0.6, 1.4)', NbTrDiscr='mult(0.5, 1.2)', ExprOpt='max(chi2)'\n",
    "* MoteurOpt='zoopt(100)' => 2021-01-14 10:52:28,040 1920 analyses => 12 results, Wall time: 3min 45s\n",
    "* MoteurOpt='zoopt(100)' => 2021-08-22 10:52:28,040 1920 analyses => 12 results, Wall time: 3min 45s\n",
    "\n",
    "**Variant 3b: \"main\"**\n",
    "\n",
    "* 12 analyses specs (12 parallel threads)\n",
    "* Default: CritChx='AIC', IntervConf=95, MethOutliers='tucquant(7)', NbTrModel='mult(0.6, 1.4)', NbTrDiscr='mult(0.5, 1.2)', ExprOpt='max(chi2)'\n",
    "* MoteurOpt='zoopt(100)' => 2021-01-14 10:30:19,463 2400 analyses => 22 results, Wall time: 4min 28s\n",
    "* MoteurOpt='zoopt(100)' => 2021-08-22 18:55:55,701 2400 analyses => 22 results, Wall time: 3min 31s\n",
    "* MoteurOpt='zoopt(100)' => 2021-11-20 11:12:31,658 2400 analyses => 22 results, Wall time: 5min\n",
    "  (Note: longer, due to new quality indicators BalQua* computation added in results post-processing in early 2021-11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zoptr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Save results for later reload or examination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.toExcel(workDir / f'valarc-mcds-optimiser{varOpt}-results.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results.toExcel(workDir / 'valarc-mcds-optimiser-results-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4B. Or : Restart optimisation from recovery file\n",
    "\n",
    "(already run above)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. MCDS Zeroth Order Truncation Optimiser object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Warning: Must be a real clone of the above one, otherwise recovery may not work !\n",
    "zoptr = ads.MCDSZerothOrderTruncationOptimiser \\\n",
    "                (dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                 transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                 sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                 abbrevCol=optAbbrevCol, abbrevBuilder=optimAbbrev,\n",
    "                 anlysIndCol=optIndCol, sampleIndCol=sampleNumCol,\n",
    "                 distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                 surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                 resultsHeadCols=dict(before=[optIndCol], sample=sampleSelCols, after=optimParamSpecCols),\n",
    "                 workDir=workDir, runMethod='os.system', runTimeOut=None, logProgressEvery=1,\n",
    "                 defEstimKeyFn=defEstimKeyFn, defEstimAdjustFn=defEstimAdjustFn,\n",
    "                 defEstimCriterion=defEstimCriterion, defCVInterval=defCVInterval,\n",
    "                 defExpr2Optimise=defExpr2Optimise, defMinimiseExpr=defMinimiseExpr,\n",
    "                 defOutliersMethod=defOutliersMethod, defOutliersQuantCutPct=defOutliersQuantCutPct,\n",
    "                 defFitDistCutsFctr=defFitDistCutsFctr, defDiscrDistCutsFctr=defDiscrDistCutsFctr,\n",
    "                 defSubmitTimes=defSubmitTimes, defSubmitOnlyBest=defSubmitOnlyBest,\n",
    "                 defCoreMaxIters=defCoreMaxIters, defCoreTermExprValue=defCoreTermExprValue,\n",
    "                 defCoreAlgorithm=defCoreAlgorithm, defCoreMaxRetries=defCoreMaxRetries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Run optimisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Optimisations with recovery\n",
    "results2 = zoptr.run(dfOptimExplSpecs, recover=True, threads=12)\n",
    "\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zoptr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Save results for later reload or examination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2.toExcel(workDir / f'valarc-mcds-optimiser{varOpt}-results2.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4C. Or : Load optimisation results from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    # An analyser object knowns how to build an empty results object ...\n",
    "    zoptr = ads.MCDSZerothOrderTruncationOptimiser \\\n",
    "                    (dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                     transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                     sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                     abbrevCol=optAbbrevCol, abbrevBuilder=optimAbbrev,\n",
    "                     anlysIndCol=optIndCol, sampleIndCol=sampleNumCol,\n",
    "                     distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                     surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                     resultsHeadCols=dict(before=[optIndCol], sample=sampleSelCols, after=optimParamSpecCols))\n",
    "    \n",
    "    # TODO: use new loadFrom param to load data !!!\n",
    "    results = zoptr.setupResults()\n",
    "    \n",
    "    # Load results from file.\n",
    "    resFileName = workDir / f'valarc-mcds-optimiser{varOpt}-results.xlsx'\n",
    "    print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "    results.fromExcel(resFileName)\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print('... {} optimisations to analyse'.format(len(results)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Deduce analyses specs from optimisation results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Short string for sample \"identification\"\n",
    "def sampleAbbrev(sSample):\n",
    "    \n",
    "    abrvSpe = ''.join(word[:4].title() for word in sSample['Espèce'].split(' ')[:2])\n",
    "    \n",
    "    sampAbbrev = '{}-{}-{}-{}'.format(abrvSpe, sSample.Passage.replace('+', ''),\n",
    "                                      sSample.Adulte.replace('+', ''), sSample['Durée'])\n",
    "    \n",
    "    return sampAbbrev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Short string for analysis \"identification\"\n",
    "def analysisAbbrev(sAnlys):\n",
    "    \n",
    "    # Sample abbreviation\n",
    "    abbrevs = [sampleAbbrev(sAnlys)]\n",
    "\n",
    "    # Model + Parameters abbreviation\n",
    "    abbrevs += [sAnlys['FonctionClé'][:3].lower(), sAnlys['SérieAjust'][:3].lower()]\n",
    "    dTroncAbrv = { 'l': 'TrGche' if 'TrGche' in sAnlys.index else 'TroncGche',\n",
    "                   'r': 'TrDrte' if 'TrDrte' in sAnlys.index else 'TroncDrte',\n",
    "                   'm': 'NbTrches' if 'NbTrches' in sAnlys.index else 'NbTrModel'\n",
    "                                   if 'NbTrModel' in sAnlys.index else  'NbTrchMod',\n",
    "                   'd': 'NbTrDiscr' }\n",
    "    for abrv, name in dTroncAbrv.items():\n",
    "        if name in sAnlys.index and not pd.isnull(sAnlys[name]):\n",
    "            abbrevs.append('{}{}'.format(abrv, sAnlys[name][0].lower() if isinstance(sAnlys[name], str)\n",
    "                                               else int(sAnlys[name])))\n",
    "   \n",
    "    return '-'.join(abbrevs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "varIndCol = 'NumAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get sample and analysis params, and above all optimised truncation param. values from optimiser results.\n",
    "optTgtCols = ['TrGche', 'TrDrte', 'NbTrchMod']\n",
    "#optTgtCols = ['TroncGche', 'TroncDrte', 'NbTrModel']\n",
    "dfAnlysSpecs = results.dfData[['Espèce', 'Passage', 'Adulte', 'Durée', 'FonctionClé', 'SérieAjust',\n",
    "                               'minDist', 'maxDist', 'fitDistCuts'] + optTgtCols].copy()\n",
    "\n",
    "# Add analysis abbreviation from truncation params optim. specs (not from optimised results).\n",
    "dfAnlysSpecs[anlysAbbrevCol] = dfAnlysSpecs.apply(analysisAbbrev, axis='columns')\n",
    "\n",
    "# No need for the truncation params optim. specs anymore\n",
    "dfAnlysSpecs.drop(columns=optTgtCols, inplace=True)\n",
    "\n",
    "# Rename optimised truncation param. columns for analysis\n",
    "dfAnlysSpecs.rename(columns=dict(minDist='TrGche', maxDist='TrDrte', fitDistCuts='NbTrchMod'), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# But non-optimised truncation parameters are not in optimiser result columns (minDist, maxDist, fitDisCuts, ...) ...\n",
    "# so we have to get them back from optimisation specs (TrGche, TrDrte, NbTrchMod, ...)\n",
    "\n",
    "# String specs are optimisation params, numerical ones are already determined truncation params.\n",
    "bdfToBeKeptSpecCells = results.dfData[optTgtCols].applymap(lambda v: isinstance(v, str))\n",
    "\n",
    "dfAnlysSpecs[optTgtCols] = dfAnlysSpecs[optTgtCols].where(bdfToBeKeptSpecCells,\n",
    "                                                          other=results.dfData[optTgtCols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "workDir = tmpDir / 'mcds-anaftopt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6A. Or : Really run analyses\n",
    "\n",
    "(now truncation parameters have been auto-computed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. MCDS Analyser object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "\n",
    "anlysr = ads.MCDSAnalyser(dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea,\n",
    "                          transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                          sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                          abbrevCol=anlysAbbrevCol, anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                          distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                          surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                          resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                               after=anlysParamCols + [anlysAbbrevCol]),\n",
    "                          workDir=workDir, logProgressEvery=1,\n",
    "                          defEstimKeyFn=defEstimKeyFn, defEstimAdjustFn=defEstimAdjustFn,\n",
    "                          defEstimCriterion=defEstimCriterion, defCVInterval=defCVInterval,\n",
    "                          defMinDist=defMinDist, defMaxDist=defMaxDist,\n",
    "                          defFitDistCuts=defFitDistCuts, defDiscrDistCuts=defDiscrDistCuts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Check analysis explicit specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    anlysr.explicitParamSpecs(dfExplParamSpecs=dfAnlysSpecs, dropDupes=True, check=True)\n",
    "\n",
    "assert len(dfAnlysSpecs) == nOptimedAnlyses, f'{len(dfAnlysSpecs)} != {nOptimedAnlyses}'\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod'], str(userParamSpecCols)\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts'], str(intParamSpecCols)\n",
    "assert unmUserParamSpecCols == []\n",
    "assert verdict\n",
    "assert not reasons, str(reasons)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Run analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Analyses : min=5, max=11s elapsed for 64 analyses with 6 threads on Windows 10, 6-core i7-8850H, PCI-e SSD, \"optimal performances\" power scheme\n",
    "# Analyses : min=2.1, max=2.5s elapsed for 22 analyses with 6-12 threads on Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme\n",
    "# Analyses : 1.5s elapsed for 12 analyses with 6-12 threads on a Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme\n",
    "results = anlysr.run(dfAnlysSpecs, threads=12)\n",
    "\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Save results for later reload or examination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.toExcel(workDir / f'valarc-mcds-analyser-afteropt{varOpt}-results.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results.toExcel(workDir / 'valarc-mcds-analyser-afteropt-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6B. Or : Load analyses results from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    # An analyser object knowns how to build an empty results object ...\n",
    "    anlysr = ads.MCDSAnalyser(dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea,\n",
    "                              resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                                   after=[anlysAbbrevCol]),\n",
    "                              transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                              sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                              abbrevCol=anlysAbbrevCol, anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                              distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                              surveyType=surveyType, distanceType=distanceType, clustering=clustering)\n",
    "    \n",
    "    results = anlysr.setupResults()\n",
    "    \n",
    "    # Load results from file.\n",
    "    resFileName = workDir / f'valarc-mcds-analyser-afteropt{varOpt}-results.xlsx'\n",
    "    print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "    results.fromExcel(resFileName)\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print('... {} analyses to compare'.format(len(results)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VIII. Truncation optimisation : Study on parameter variants\n",
    "\n",
    "Objective: How to choose key parameters ?\n",
    "* how many outliers ?\n",
    "* how many max iters ?\n",
    "* correlation with number of sightings ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.express as plyx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data set, samples, transects, analyses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Warning: First run [VII. Truncation optimisation (short code / fast run)](#VII.-Truncation-optimisation-(short-code-and-fast-run)) 0, 1 and 2 above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOptimExplSpecs = ads.Analyser.explicitVariantSpecs('refin/ACDC2019-Naturalist-ExtraitSpecsOptanalyses.xlsx', \n",
    "                                                     ignore=['Params1_expl', 'Params2_expl'])\n",
    "\n",
    "dfOptimExplSpecs.drop(dfOptimExplSpecs[dfOptimExplSpecs[['TrGche', 'TrDrte', 'NbTrchMod', 'MultiOpt']]\n",
    "                                           .isnull().all(axis='columns')].index,\n",
    "                      inplace=True)\n",
    "\n",
    "dfOptimExplSpecs.drop(columns=['TrGche', 'TrDrte', 'NbTrchMod', 'MultiOpt'], inplace=True)\n",
    "\n",
    "nOptimExplSpecs = len(dfOptimExplSpecs)\n",
    "\n",
    "dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Parameter variants plan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nTimes = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expr2MaxPlan = ['chi2', 'ks']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliersPctPlan = [2.5, 5.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxItersPlan = [50, 100, 150, 200, 250, 300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3A. Or: Run optimisations according to the plan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nParSets = len(expr2MaxPlan) * len(outliersPctPlan) * len(maxItersPlan)\n",
    "nOpt2Run = len(dfOptimExplSpecs) * nTimes * nParSets\n",
    "print(f'About to run {nOpt2Run} optimisations !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes de dfOptimExplSpecs donnant les paramètres d'analyse / optimisation\n",
    "optimParamsSpecsCols  = ['FonctionClé', 'SérieAjust', 'CritChx', 'IntervConf',\n",
    "                         'TroncGche', 'TroncDrte', 'MethOutliers', 'NbTrModel', 'NbTrDiscr',\n",
    "                         'ExprOpt', 'ParExec', 'MoteurOpt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "ldfResults = list()\n",
    "\n",
    "nParSetInd = 1\n",
    "for expr2Max in expr2MaxPlan:\n",
    "    \n",
    "    for olrsPct in outliersPctPlan:\n",
    "\n",
    "        for maxIters in maxItersPlan:\n",
    "\n",
    "            logger.info(f'Params set {nParSetInd}/{nParSets}: {expr2Max=}, {nTimes=}, {maxIters=}, {olrsPct=:.1f}')\n",
    "            \n",
    "            # Prepare optim. params.\n",
    "            dfMoreOptimCols = \\\n",
    "                pd.DataFrame([dict(CritChx='AIC', IntervConf=95,\n",
    "                                   TroncGche='auto', TroncDrte='auto',\n",
    "                                   MethOutliers=f'tucquant({olrsPct:.1f})',\n",
    "                                   NbTrModel='mult(2/3, 3/2)', NbTrDiscr=None,\n",
    "                                   ExprOpt=f'max({expr2Max})', ParExec=f'times({nTimes})',\n",
    "                                   MoteurOpt=f'zoopt({maxIters})')]*len(dfOptimExplSpecs))\n",
    "\n",
    "            dfOptVarExplSpecs = pd.concat([dfOptimExplSpecs.reset_index(drop=True), dfMoreOptimCols], axis='columns')\n",
    "\n",
    "            # Run optimisation.\n",
    "            zoptr = ads.MCDSZerothOrderTruncationOptimiser \\\n",
    "                            (dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea, \n",
    "                             transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                             sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                             abbrevCol=optAbbrevCol, abbrevBuilder=optimAbbrev,\n",
    "                             anlysIndCol=optIndCol, sampleIndCol=sampleNumCol,\n",
    "                             distanceUnit='Meter', areaUnit='Hectare',\n",
    "                             surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                             resultsHeadCols=dict(before=[optIndCol], sample=sampleSelCols, after=optimParamsSpecsCols),\n",
    "                             workDir='/tmp', logData=False,                 \n",
    "                             defCoreMaxIters=120)\n",
    "\n",
    "            results = zoptr.run(dfOptVarExplSpecs, threads=12)\n",
    "\n",
    "            zoptr.shutdown()\n",
    "\n",
    "            # Save results for this run\n",
    "            ldfResults.append(results.dfData)\n",
    "            \n",
    "            nParSet += 1\n",
    "        \n",
    "# Done : concat and save results.\n",
    "dfResults = pd.concat(ldfResults, ignore_index=True)\n",
    "\n",
    "resFileName = 'tmp/valarc-mcds-opter-res4stats.xlsx'\n",
    "dfResults.to_excel(resFileName, index=False)\n",
    "logger.info(f'Results saved to {resFileName}')\n",
    "\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3B. Or : Load results from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'computed' not in dir():\n",
    "    computed = False\n",
    "if not computed:\n",
    "    \n",
    "    # Load results from file.\n",
    "    #resFileName = 'tmp/valarc-mcds-opter-res4stats-20200705.xlsx'\n",
    "    resFileName = 'tmp/valarc-mcds-opter-res4stats-20201103.xlsx'\n",
    "    print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "    dfResults = pd.read_excel(resFileName)\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print('... {} results to process'.format(len(dfResults)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. First stats on optimisation results\n",
    "\n",
    "* raw stats : mean and std\n",
    "* first correlations : number of analyses / optimised criterium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfResults), dfResults.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfResults.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optResCols = ['minDist', 'maxDist', 'fitDistCuts', 'chi2', 'ks', 'chi2*ks']\n",
    "#groupCols = [col for col in dfResults.columns if col not in optResCols]\n",
    "groupCols = ['Espèce', 'Passage', 'Adulte', 'Durée', 'FonctionClé', 'SérieAjust', 'MethOutliers', 'ExprOpt',\n",
    "             'MinDist', 'MaxDist', 'FitDistCuts', 'NFunEvals']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Raw stats : mean, std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfStats = dfResults.groupby(groupCols).agg(['mean', 'std'])\n",
    "dfStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resFileName = 'tmp/valarc-mcds-opter-stats.xlsx'\n",
    "dfStats.reset_index().to_excel(resFileName)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Visual correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfResults['NFunEvalsR'] = dfResults.NFunEvals.apply(lambda v: int(50*np.ceil(v/50)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for esp in dfResults['Espèce'].unique():\n",
    "    axes = dfResults[dfResults['Espèce'] == esp].plot.hexbin(y='NFunEvalsR', x='chi2', gridsize=(20, 6), figsize=(14, 3))\n",
    "    axes.set_title(f'{esp} : chi2 / NFunEvals')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for esp in dfResults['Espèce'].unique():\n",
    "    axes = dfResults[dfResults['Espèce'] == esp].plot.hexbin(y='NFunEvalsR', x='ks', gridsize=(20, 6), figsize=(14, 3))\n",
    "    axes.set_title(f'{esp} : ks / NFunEvals')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfResults['Outliers'] = dfResults.MethOutliers.apply(lambda s: float(s[len('tucquant('):-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = dfResults.plot.scatter(y='Outliers', x='chi2', figsize=(14, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = dfResults.plot.scatter(y='Outliers', x='ks', figsize=(14, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfResults.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plyx.violin(dfResults, x='chi2', y='NFunEvalsR', facet_row='Outliers', color=\"Espèce\", orientation='h', height=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plyx.violin(dfResults, x='ks', y='NFunEvalsR', facet_row='Outliers', color=\"Espèce\", orientation='h', height=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Computed correlations\n",
    "\n",
    "(linéaires, de Pearson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pearsonCorr(df, x, y):\n",
    "    \n",
    "    cv = np.cov(df[x].values, df[y].values)\n",
    "    \n",
    "    return pd.Series(dict(corr=cv[0, 1] / cv[0, 0] / cv[1, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nb of analyses run / optimisation criterium\n",
    "groupCols = ['Espèce', 'Passage', 'Adulte', 'Durée', 'FonctionClé', 'SérieAjust', 'MethOutliers', 'ExprOpt']\n",
    "\n",
    "df = dfResults.loc[dfResults.ExprOpt == 'max(chi2)',\n",
    "                   groupCols + ['NFunEvals', 'chi2']].groupby(groupCols).apply(pearsonCorr, x='NFunEvals', y='chi2')\n",
    "df.rename(columns=dict(corr='NFun/Expr'), inplace=True)\n",
    "dfCorr = df.copy()\n",
    "\n",
    "df = dfResults.loc[dfResults.ExprOpt == 'max(ks)',\n",
    "                   groupCols + ['NFunEvals', 'ks']].groupby(groupCols).apply(pearsonCorr, x='NFunEvals', y='ks')\n",
    "df.rename(columns=dict(corr='NFun/Expr'), inplace=True)\n",
    "dfCorr = dfCorr.append(df)\n",
    "\n",
    "dfCorr.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfCorr[dfCorr.index.get_level_values('ExprOpt') == 'max(ks)'].sort_values(by='NFun/Expr', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfCorr[dfCorr.index.get_level_values('ExprOpt') == 'max(chi2)'].sort_values(by='NFun/Expr', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# % of outliers excluded / optimisation criterium\n",
    "groupCols = ['Espèce', 'Passage', 'Adulte', 'Durée']\n",
    "\n",
    "df = dfResults.loc[dfResults.ExprOpt == 'max(chi2)',\n",
    "                   groupCols + ['Outliers', 'chi2']].groupby(groupCols).apply(pearsonCorr, x='Outliers', y='chi2')\n",
    "df.rename(columns=dict(corr='Outliers/Expr'), inplace=True)\n",
    "dfCorr = df.copy()\n",
    "\n",
    "df = dfResults.loc[dfResults.ExprOpt == 'max(ks)',\n",
    "                   groupCols + ['Outliers', 'ks']].groupby(groupCols).apply(pearsonCorr, x='Outliers', y='ks')\n",
    "df.rename(columns=dict(corr='Outliers/Expr'), inplace=True)\n",
    "dfCorr = dfCorr.append(df)\n",
    "\n",
    "dfCorr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Run analyses with optimised truncations\n",
    "\n",
    "(to get the actual numbers of sightings retained)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Deduce analyses specs from optimisation results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfResults.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "varIndCol = 'NumAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get sample and analysis params, and above all optimised truncation param. values from optimiser results.\n",
    "#optTgtCols = ['TrGche', 'TrDrte', 'NbTrchMod']\n",
    "optTgtCols = ['TroncGche', 'TroncDrte', 'NbTrModel']\n",
    "otherOptTgtCols = ['Outliers', 'NFunEvals']\n",
    "dfAnlysSpecs = dfResults[['Espèce', 'Passage', 'Adulte', 'Durée', 'FonctionClé', 'SérieAjust',\n",
    "                          'minDist', 'maxDist', 'fitDistCuts'] + optTgtCols + otherOptTgtCols].copy()\n",
    "\n",
    "# Add analysis abbreviation from truncation params optim. specs (not from optimised results).\n",
    "dfAnlysSpecs[anlysAbbrevCol] = dfAnlysSpecs.apply(analysisAbbrev, axis='columns')\n",
    "\n",
    "# No need for the truncation params optim. specs anymore\n",
    "dfAnlysSpecs.drop(columns=optTgtCols, inplace=True)\n",
    "\n",
    "# Rename optimised truncation param. columns for analysis\n",
    "dfAnlysSpecs.rename(columns=dict(minDist='TrGche', maxDist='TrDrte', fitDistCuts='NbTrchMod'), inplace=True)\n",
    "\n",
    "dfAnlysSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "workDir = tmpDir / 'mcds-optstats'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "computed = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Or : Really run analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i. MCDS Analyser object\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "\n",
    "anlysr = ads.MCDSAnalyser(dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea,\n",
    "                          transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                          sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                          abbrevCol=anlysAbbrevCol, anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                          distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                          surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                          resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                               after=anlysParamCols + [anlysAbbrevCol, 'Outliers', 'NFunEvals']),\n",
    "                          workDir=workDir,\n",
    "                          defEstimKeyFn=defEstimKeyFn, defEstimAdjustFn=defEstimAdjustFn,\n",
    "                          defEstimCriterion=defEstimCriterion, defCVInterval=defCVInterval,\n",
    "                          defMinDist=defMinDist, defMaxDist=defMaxDist,\n",
    "                          defFitDistCuts=defFitDistCuts, defDiscrDistCuts=defDiscrDistCuts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ii. Check analysis explicit specs\n",
    "dfAnlysSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    anlysr.explicitParamSpecs(dfExplParamSpecs=dfAnlysSpecs, dropDupes=True, check=True)\n",
    "\n",
    "assert len(dfAnlysSpecs) == len(dfResults)\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod'], str(userParamSpecCols)\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts'], str(intParamSpecCols)\n",
    "assert unmUserParamSpecCols == []\n",
    "assert verdict\n",
    "assert not reasons, str(reasons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# iii. Run analyses\n",
    "\n",
    "# Analyses : 20mn for 8640 analyses with 12 threads on Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme\n",
    "results = anlysr.run(dfAnlysSpecs, threads=12)\n",
    "\n",
    "computed = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iiii. Save results for later reload or examination\n",
    "results.toExcel(workDir / 'valarc-mcds-analyser-afteropt-results.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#results.toExcel(workDir / 'valarc-mcds-analyser-afteropt-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Or : Load analyses from a previous run\n",
    "\n",
    "(already run and saved above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not computed:\n",
    "    \n",
    "    # An analyser object knowns how to build an empty results object ...\n",
    "    anlysr = ads.MCDSAnalyser(dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea,\n",
    "                              resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                                   after=[anlysAbbrevCol]),\n",
    "                              transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                              sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                              abbrevCol=anlysAbbrevCol, anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                              distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                              surveyType=surveyType, distanceType=distanceType, clustering=clustering)\n",
    "    \n",
    "    results = anlysr.setupResults()\n",
    "    \n",
    "    # Load results from file.\n",
    "    resFileName = workDir / 'valarc-mcds-analyser-afteropt-results.xlsx'\n",
    "    print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "    results.fromExcel(resFileName)\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print('Just computed, not reloading ...')\n",
    "    \n",
    "print('... {} analyses to study'.format(len(results)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnRes = results.dfTransData('fr')\n",
    "dfAnRes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Other stats on analysis results\n",
    "\n",
    "Through NObs mainly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnRes.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for esp in dfAnRes['Espèce'].unique():\n",
    "    axes = dfAnRes[dfAnRes['Espèce'] == esp].plot.hexbin(y='NObs', x='Chi2 P', gridsize=(20, 6), figsize=(14, 3))\n",
    "    axes.set_title(f'{esp} : chi2 / NObs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for esp in dfAnRes['Espèce'].unique():\n",
    "    axes = dfAnRes[dfAnRes['Espèce'] == esp].plot.hexbin(y='NObs', x='KS P', gridsize=(20, 6), figsize=(14, 3))\n",
    "    axes.set_title(f'{esp} : KS / NObs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnRes['NFunEvalsR'] = dfAnRes.NFunEvals.apply(lambda v: int(50*np.ceil(v/50)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plyx.scatter(dfAnRes[['Espèce', 'Chi2 P', 'NObs', 'Outliers', 'NFunEvalsR']].dropna(subset=['NObs']),\n",
    "             x='Chi2 P', y='NObs', facet_col='Outliers', facet_row='NFunEvalsR', color='Espèce', height=1200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plyx.scatter(dfAnRes[['Espèce', 'KS P', 'NObs', 'Outliers', 'NFunEvalsR']].dropna(subset=['NObs']),\n",
    "             x='KS P', y='NObs', facet_col='Outliers', facet_row='NFunEvalsR', color='Espèce', height=1200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plyx.scatter(dfAnRes[['Espèce', 'Chi2 P', 'NObs', 'Outliers', 'NFunEvals']].dropna(subset=['NObs']),\n",
    "            x='NObs', y='Chi2 P', facet_col='Outliers', facet_row='Espèce', color='NFunEvals', height=700)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plyx.scatter(dfAnRes[['Espèce', 'KS P', 'NObs', 'Outliers', 'NFunEvals']].dropna(subset=['NObs']),\n",
    "             x='NObs', y='KS P', facet_col='Outliers', facet_row='Espèce', color='NFunEvals', height=700)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py38]",
   "language": "python",
   "name": "conda-env-py38-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
