{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!-- Auto table of contents -->\n",
    "<h1 class='tocIgnore'>Development archives: auto-filtered and sorted results / reports</h1>\n",
    "\n",
    "**pyaudisam**: Automation of Distance Sampling analyses with [Distance software](http://distancesampling.org/)\n",
    "\n",
    "Copyright (C) 2021 Jean-Philippe Meuret\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify it under the terms\n",
    "of the GNU General Public License as published by the Free Software Foundation,\n",
    "either version 3 of the License, or (at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n",
    "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n",
    "See the GNU General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU General Public License along with this program.\n",
    "If not, see https://www.gnu.org/licenses/.\n",
    "\n",
    "<div style=\"overflow-y: auto\">\n",
    "  <h2 class='tocIgnore'>Table of contents</h2>\n",
    "  <div id=\"toc\"></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "$.getScript('ipython_notebook_toc.js')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import pathlib as pl\n",
    "\n",
    "import re\n",
    "\n",
    "import concurrent.futures as cofu\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activate Warnings as Exception\n",
    "#import warnings\n",
    "#warnings.filterwarnings('error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyaudisam as ads\n",
    "\n",
    "ads.runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create temporary directory if not yet done.\n",
    "tmpDir = pl.Path('tmp')\n",
    "tmpDir.mkdir(exist_ok=True)\n",
    "\n",
    "tmpDir.absolute().as_posix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logging configuration.\n",
    "ads.log.configure(handlers=[sys.stdout, tmpDir / 'devarc2.log'], reset=True,\n",
    "                  loggers=[dict(name='matplotlib', level=ads.WARNING),\n",
    "                           dict(name='ads', level=ads.INFO),\n",
    "                           dict(name='ads.dat', level=ads.INFO2),\n",
    "                           dict(name='ads.eng', level=ads.INFO2),\n",
    "                           dict(name='ads.onr', level=ads.DEBUG),\n",
    "                           dict(name='ads.anr', level=ads.DEBUG1)])\n",
    "\n",
    "logger = ads.logger('devarc2', level=ads.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Commons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# v. Sample and analysis identification.\n",
    "def sampleAbbrev(sSamp):\n",
    "    abbrvs = [''.join(word[:4].title() for word in sSamp[colEspece].split(' ')[:2])]\n",
    "    if colPassage in sSamp.index and not pd.isnull(sSamp.Passage) and sSamp.Passage:\n",
    "        abbrvs.append(sSamp.Passage.replace('+', ''))\n",
    "    if 'Durée' in sSamp.index:\n",
    "        abbrvs.append(sSamp['Durée'].replace('+', ''))\n",
    "    if 'Adulte' in sSamp.index:\n",
    "        abbrvs.append(sSamp.Adulte.replace('+', ''))\n",
    "    return '-'.join(abbrvs)\n",
    "\n",
    "def analysisAbbrev(sAnlys):\n",
    "    \n",
    "    # Sample abbreviation\n",
    "    abbrevs = [sampleAbbrev(sAnlys)]\n",
    "\n",
    "    # Model + Parameters abbreviation\n",
    "    abbrevs += [sAnlys['FonctionClé'][:3].lower(), sAnlys['SérieAjust'][:3].lower()]\n",
    "    dTroncAbbrv = { 'l': 'TrGche' if 'TrGche' in sAnlys.index else 'TroncGche',\n",
    "                    'r': 'TrDrte' if 'TrDrte' in sAnlys.index else 'TroncDrte',\n",
    "                    'm': 'NbTrModel' if 'NbTrModel' in sAnlys.index else  'NbTrchMod',\n",
    "                    'd': 'NbTrDiscr' }\n",
    "    for abbrev, name in dTroncAbbrv.items():\n",
    "        if name in sAnlys.index and not pd.isnull(sAnlys[name]):\n",
    "            abbrevs.append('{}{}'.format(abbrev, sAnlys[name][0].lower() if isinstance(sAnlys[name], str)\n",
    "                                                 else int(sAnlys[name])))\n",
    "   \n",
    "    return '-'.join(abbrevs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load optanalysis results\n",
    "\n",
    "Tooling for subsequent tests, comparison, etc ... (see below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select target study\n",
    "dossier = pl.Path('../../perso/donnees/acdc')\n",
    "\n",
    "nomEtude = 'ACDC2019'\n",
    "sousEtude = '-Nat'\n",
    "#sousEtude = '-Pap'\n",
    "\n",
    "varEtude = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Select results file to load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List eligible folders for choosen study\n",
    "resFileName = f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-resultats.xlsx'\n",
    "\n",
    "resFolders = [fn.name for fn in dossier.glob('[0-9]'*6+'-'+'[0-9]'*4) if (fn / resFileName).is_file()]\n",
    "\n",
    "logger.info('Rapports historiques disponibles: {}'.format(', '.join(resFolders)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select the one to process\n",
    "workDir = dossier / resFolders[0]  # <=== Choisir le dossier de résultats ici.\n",
    "\n",
    "updatedResFileNameExists = (pl.Path('tmp') / resFileName).is_file()\n",
    "if not updatedResFileNameExists:\n",
    "\n",
    "    resFileName = workDir / resFileName\n",
    "\n",
    "    logger.info(f'Fichier choisi : {resFileName.as_posix()}')\n",
    "    \n",
    "else:\n",
    "    \n",
    "    # An updated results file (with sample stats : see below) exists: select this one.\n",
    "    resFileName = tmpDir / resFileName\n",
    "\n",
    "    logger.info(f'... mais résultats à jour aussi: {resFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Build an MCDSTruncOptanalysisResultsSet object to load file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build an empty MCDSTruncOptanalysisResultsSet from an MCDSTruncationOptanalyser object\n",
    "\n",
    "# i. Load individualised data with computed distances + transects info.\n",
    "fpn = dossier / f'{nomEtude}{sousEtude}-ObsIndivDist.xlsx'\n",
    "with pd.ExcelFile(fpn) as xlsFile:\n",
    "    dfObsCatIndiv = pd.read_excel(xlsFile, sheet_name='Donnees')\n",
    "    dfTransects = pd.read_excel(xlsFile, sheet_name='Inventaires')\n",
    "\n",
    "print(dict(etude=nomEtude+sousEtude, donnees=len(dfObsCatIndiv), inventaires=len(dfTransects)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ii. Data description\n",
    "colEspece = 'Espèce'\n",
    "colPassage = 'Passage'\n",
    "colDistance = 'Distance'\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = colPassage\n",
    "\n",
    "assert 'effortCol' not in dir() or effortCol == 'Effort'  # In rare cases, needs to be defined before here, but the same way !\n",
    "effortCol = 'Effort'\n",
    "\n",
    "colsSpeSelEchant = ['Adulte', 'Durée']  # Colonnes de sélection des échantillons : en plus de Espèce et Passage. \n",
    "sampleDistCol = colDistance\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'Echant'\n",
    "sampleSelCols = [colEspece, passIdCol] + colsSpeSelEchant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iii. Analysis params.\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Sq. Kilometer'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "\n",
    "groupage = False\n",
    "effortConst = 1 # Const effort value = 1 per pass on each points\n",
    "\n",
    "dZoneEtude = dict(Zone='ACDC', Surface=24) # Area unit = Sq. Kilometer (see areaUnit above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iv. Opt-analyses parameters.\n",
    "anlysIndCol = 'Analyse'\n",
    "anlysAbbrevCol = 'Abrev. Analyse'\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vi. Distance truncation grouping parameters (for best analysis selection after some simplification)\n",
    "ldTruncIntrvSpecs = [dict(col='left', minDist=5.0, maxLen=5.0),  dict(col='right', minDist=25.0, maxLen=25.0)]\n",
    "truncIntrvEpsilon = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vii. At last, the empty results set object !\n",
    "optanlr = \\\n",
    "    ads.MCDSTruncationOptanalyser(dfObsCatIndiv, dfTransects=dfTransects,\n",
    "                                  effortConstVal=effortConst, dSurveyArea=dZoneEtude, \n",
    "                                  transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                                  sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                                  abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                                  anlysIndCol=anlysIndCol, sampleIndCol=sampleNumCol,\n",
    "                                  distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                                  surveyType=surveyType, distanceType=distanceType, clustering=groupage,\n",
    "                                  ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon,\n",
    "                                  resultsHeadCols=dict(before=[anlysIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                                       after=anlysParamCols + [anlysAbbrevCol]))\n",
    "\n",
    "results = optanlr.setupResults()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Load file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results.fromFile(resFileName, postComputed=False)  # Use postComputed=True to avoid updating post-computed columns on load."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Backward compatibility: complete columns set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not updatedResFileNameExists and 'sample stats' not in results._dfData.columns.unique(level=0):\n",
    "    \n",
    "    # If needed, add sample stats a posteriori\n",
    "    # (these stats had not been implemented when the historical results were saved to disk)\n",
    "    dfSampleStats = pd.read_excel(dossier / f'{nomEtude}{sousEtude}-StatsEchantillons.xlsx')\n",
    "    dfSampleStats.rename(columns={'NTot Obs': 'NTot Obs0'}, inplace=True)\n",
    "    dfSampleStats.insert(dfSampleStats.columns.to_list().index('Distance Min'), 'NTot Obs', dfSampleStats['NTot Obs0'])\n",
    "    dfSampleStats.drop(columns=['NTot Obs0'], inplace=True)\n",
    "\n",
    "    miSampleCols = pd.MultiIndex.from_tuples([('header (sample)', colEspece, 'Value'),\n",
    "                                              ('header (sample)', colPassage, 'Value'),\n",
    "                                              ('header (sample)', colsSpeSelEchant[0], 'Value'),\n",
    "                                              ('header (sample)', colsSpeSelEchant[1], 'Value')])\n",
    "    dfSampleStats.columns = miSampleCols.append(ads.MCDSEngine.MIStatSampCols)\n",
    "\n",
    "    results.dfData = results._dfData.join(dfSampleStats.set_index(miSampleCols.to_list()), on=miSampleCols.to_list())\n",
    "        \n",
    "    # Save updated results for later use.\n",
    "    logger.info('Ecriture des resultats mis à jour: {} ...'.format((tmpDir / resFileName.name).as_posix()))\n",
    "    results.toExcel(tmpDir / resFileName.name)  # Note: This actually triggers post-computations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# In case not yet done, trigger post-compute columns (re)-computation (create/update quality indicators and sort orders).\n",
    "dfActRes = results.dfTransData('fr')\n",
    "dfActRes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Figures and checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some facts and figures\n",
    "dfActRes['NbTot Pars'].value_counts(), dfActRes['NbPars FnClé'].value_counts(), dfActRes['NbPars SérAjust'].value_counts(), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some checks before going on.\n",
    "assert dfActRes.Analyse.nunique() == len(dfActRes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load / generate filter & sort reports\n",
    "\n",
    "Tooling for subsequent tests, comparison, etc ... (see below).\n",
    "\n",
    "Early 2021 prototype version, or industrialised version, ... for comparison, non-regression tests, quality tests ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Or: Load early 2021 prototype report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select target study\n",
    "dossier = pl.Path('../../perso/donnees/acdc')\n",
    "\n",
    "nomEtude = 'ACDC2019'\n",
    "sousEtude = '-Nat'\n",
    "#sousEtude = '-Pap'\n",
    "\n",
    "varEtude = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select target report\n",
    "refRepFileName = f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-raptousech.ods'\n",
    "\n",
    "resFolders = [fn.name for fn in dossier.glob('[0-9]'*6+'-'+'[0-9]'*4) if (fn / refRepFileName).is_file()]\n",
    "\n",
    "logger.info('Rapports prototypes disponibles : ' + ', '.join(resFolders))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "workDir = dossier / resFolders[0]  # <=== Choisir le dossier de résultats ici.\n",
    "\n",
    "refRepFileName = workDir / refRepFileName\n",
    "\n",
    "logger.info(f'Choix {refRepFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Load report (all sheets)\n",
    "logger.info(f'Lecture du rapport prototype choisi ...')\n",
    "\n",
    "ddfProtoRep = pd.read_excel(refRepFileName, sheet_name=None)\n",
    "\n",
    "protoSheetPrefix = ''\n",
    "\n",
    "print('=> feuilles : ' + ', '.join(ddfProtoRep.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns from prototype scheme to the industrial one to compare with later if needed.\n",
    "# TODO: Add EN version\n",
    "DRefRep2IndusResCols = {'Fn Clé Mod': 'FonctionClé', 'Sér Ajust Mod': 'SérieAjust',\n",
    "                        'Dist Tronc Gche': 'TrGche', 'Dist Tronc Drte': 'TrDrte',\n",
    "                        'Tranch Dist Mod': 'NbTrchMod',\n",
    "                        \n",
    "                        'Sélection Qual Equi': 'Pré-sélection Qual Equi 1',\n",
    "                        \n",
    "                        'Qual Equi': 'Qual Equi 1',\n",
    "                        'Qual Chi2': 'Qual Chi2+',\n",
    "                        'Qual DCV': 'Qual DCv+',\n",
    "                        'Qual KS': 'Qual KS+',\n",
    "                        \n",
    "                        'Grp Dist Tronc Gche': 'Groupe Tronc Gche',\n",
    "                        'Grp Dist Tronc Drte': 'Groupe Tronc Drte',\n",
    "                        \n",
    "                        'Meil AIC Tronc Id': 'Ordre Tronc Ident AIC',\n",
    "                        \n",
    "                        'Meil CKCv Tronc Proch'     : 'Ordre Tronc Proch Chi2 KS DCv',\n",
    "                        'Meil CVDens Tronc Proch'   : 'Ordre Tronc Proch DCv',\n",
    "                        'Meil Qual Equi Tronc Proch': 'Ordre Tronc Proch Qual Equi 1',\n",
    "                        'Meil Qual Chi2 Tronc Proch': 'Ordre Tronc Proch Qual Equi Chi2+',\n",
    "                        'Meil Qual KS Tronc Proch'  : 'Ordre Tronc Proch Qual Equi KS+',\n",
    "                        'Meil Qual DCV Tronc Proch' : 'Ordre Tronc Proch Qual Equi DCv+',\n",
    "                        \n",
    "                        'Ord CKCv'     : 'Ordre Global Chi2 KS DCv',\n",
    "                        'Ord Qual Equi': 'Ordre Global Qual Equi 1',\n",
    "                        'Ord Qual Chi2': 'Ordre Global Qual Equi Chi2+',\n",
    "                        'Ord Qual KS'  : 'Ordre Global Qual Equi KS+',\n",
    "                        'Ord Qual DCV' : 'Ordre Global Qual Equi DCv+',\n",
    "                        'Ord Simpl Tronc': 'Ordre Global DeltaAIC Chi2 KS DCv'}\n",
    "\n",
    "for sn in ddfProtoRep.keys():\n",
    "    \n",
    "    if sn in ['paramètres', 'échantillons']:\n",
    "        continue\n",
    "    \n",
    "    dfFSSheet = ddfProtoRep[sn]\n",
    "    assert all(col in dfFSSheet.columns for col in DRefRep2IndusResCols), \\\n",
    "           ', '.join(col for col in DRefRep2IndusResCols if col not in dfFSSheet.columns)\n",
    "\n",
    "    dfFSSheet.rename(columns=DRefRep2IndusResCols, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Or: Load existing industrialised report\n",
    "\n",
    "Warning: May not be up-to-date with analysis results post-computation and filter-sorting evolutions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select target study\n",
    "dossier = pl.Path('../../perso/donnees/acdc')\n",
    "\n",
    "nomEtude = 'ACDC2019'\n",
    "sousEtude = '-Nat'\n",
    "#sousEtude = '-Pap'\n",
    "\n",
    "varEtude = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repFileNameGlobs = [f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-rapport.xlsx',\n",
    "                    f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-rapport*.ods']\n",
    "\n",
    "possRepFilePaths = list()\n",
    "for fnGlob in repFileNameGlobs:\n",
    "    possRepFilePaths += [fpn for fpn in dossier.glob('[0-9]'*6 + '-' + '[0-9]'*4 + '/' + fnGlob)]\n",
    "\n",
    "logger.info('Rapports industrialisés disponibles :')\n",
    "for ind, fpn in enumerate(possRepFilePaths):\n",
    "    logger.info(f'#{ind} {fpn.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repFileName = possRepFilePaths[0]  # <=== Choisir le fichier de rapport ici.\n",
    "\n",
    "workDir = repFileName.parent\n",
    "\n",
    "logger.info(f'Choix {repFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Load report (all sheets)\n",
    "logger.info(f'Lecture du rapport industrialisé choisi ...')\n",
    "\n",
    "ddfIndusRep = pd.read_excel(repFileName, sheet_name=None)\n",
    "\n",
    "indusSheetPrefix = 'MFTA-'\n",
    "\n",
    "logger.info('=> feuilles : ' + ', '.join(ddfIndusRep.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jump to [MCDSTruncOptAnalysisResultsSet non-regressions tests and checks](#MCDSTruncOptAnalysisResultsSet-non-regressions-tests-and-checks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Or: Generate industrialised filter & sort report from existing results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Select and load source results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To do so, run [Load optanalysis results](#Load-optanalysis-results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Apply filter and sort schemes for the report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define appliable filter and sort schemes\n",
    "# Warning: Carefully check all parameters in case you are targeting some kind of comparison ...\n",
    "R = ads.MCDSTruncOptanalysisResultsSet\n",
    "\n",
    "whichBestQua = [R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaBal3,  # <= WARNING: Which QuaBal<n> ?\n",
    "                R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv]\n",
    "\n",
    "whichFinalQua = R.CLCmbQuaBal3  # <= WARNING: Which QuaBal<n> ?\n",
    "ascFinalQua = False\n",
    "\n",
    "dupSubset = [R.CLNObs, R.CLEffort, R.CLDeltaAic, R.CLChi2, R.CLKS, R.CLCvMUw, R.CLCvMCw, R.CLDCv, \n",
    "             R.CLPDetec, R.CLPDetecMin, R.CLPDetecMax, R.CLDensity, R.CLDensityMin, R.CLDensityMax]\n",
    "dDupRounds = {R.CLDeltaAic: 1, R.CLChi2: 2, R.CLKS: 2, R.CLCvMUw: 2, R.CLCvMCw: 2, R.CLDCv: 2, \n",
    "              R.CLPDetec: 3, R.CLPDetecMin: 3, R.CLPDetecMax: 3, R.CLDensity: 2, R.CLDensityMin: 2, R.CLDensityMax: 2}\n",
    "\n",
    "DFilSorRepSchemes = [dict(method=R.filterSortOnExecCode,\n",
    "                          deduplicate=dict(dupSubset=dupSubset, dDupRounds=dDupRounds),\n",
    "                          filterSort=dict(whichFinalQua=whichFinalQua, ascFinalQua=ascFinalQua),\n",
    "                          preselCols=[R.CLCmbQuaBal1, R.CLCmbQuaBal2, R.CLCmbQuaBal3],\n",
    "                          preselThrhs=0.2, preselAscs=False, preselNum=15),\n",
    "                     dict(method=R.filterSortOnExCAicMulQua,\n",
    "                          deduplicate=dict(dupSubset=dupSubset, dDupRounds=dDupRounds),\n",
    "                          filterSort=dict(sightRate=92.5, nBestAIC=3, nBestQua=1, whichBestQua=whichBestQua,\n",
    "                                          nFinalRes=12, whichFinalQua=whichFinalQua, ascFinalQua=ascFinalQua),\n",
    "                          preselCols=[R.CLCmbQuaBal1, R.CLCmbQuaBal2, R.CLCmbQuaBal3],\n",
    "                          preselThrhs=0.2, preselAscs=False, preselNum=12),\n",
    "                     dict(method=R.filterSortOnExCAicMulQua,\n",
    "                          deduplicate=dict(dupSubset=dupSubset, dDupRounds=dDupRounds),\n",
    "                          filterSort=dict(sightRate=95, nBestAIC=2, nBestQua=1, whichBestQua=whichBestQua,\n",
    "                                          nFinalRes=10, whichFinalQua=whichFinalQua, ascFinalQua=ascFinalQua),\n",
    "                          preselCols=[R.CLCmbQuaBal1, R.CLCmbQuaBal2, R.CLCmbQuaBal3],\n",
    "                          preselThrhs=0.2, preselAscs=False, preselNum=10),\n",
    "                     dict(method=R.filterSortOnExCAicMulQua,\n",
    "                          deduplicate=dict(dupSubset=dupSubset, dDupRounds=dDupRounds),\n",
    "                          filterSort=dict(sightRate=97.5, nBestAIC=2, nBestQua=1, whichBestQua=whichBestQua,\n",
    "                                          nFinalRes=8, whichFinalQua=whichFinalQua, ascFinalQua=ascFinalQua),\n",
    "                          preselCols=[R.CLCmbQuaBal1, R.CLCmbQuaBal2, R.CLCmbQuaBal3],\n",
    "                          preselThrhs=0.2, preselAscs=False, preselNum=8)]\n",
    "\n",
    "results.filSorCache.clear()\n",
    "results.dFilSorSchemes.clear()\n",
    "\n",
    "[results.filSorSchemeId(scheme) for scheme in DFilSorRepSchemes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Apply target filter and sort schemes\n",
    "ddfIndusRep = dict()\n",
    "dIndusFSSteps = dict()\n",
    "for scheme in DFilSorRepSchemes:\n",
    "    filSorSchId, dfFilSorRes, filSorSteps = \\\n",
    "        results.dfFilSorData(scheme=scheme, columns=None, lang='fr', rebuild=False)\n",
    "    ddfIndusRep[filSorSchId] = dfFilSorRes\n",
    "    dIndusFSSteps[filSorSchId] = filSorSteps\n",
    "    \n",
    "indusSheetPrefix = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add full results sheet (just as in real filter & sort reports)\n",
    "ddfIndusRep['Détails'] = results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(', '.join(ddfIndusRep.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MCDSTruncOptAnalysisResultsSet non regressions tests and checks\n",
    "\n",
    "To do so, run [Load optanalysis results](#Load-optanalysis-results) or [Load / generate filter & sort reports](#Load-%2F-generate-filter-%26-sort-reports) for each results to load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load results to compare / check"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Reference results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or: Load results from _RESULTS_ file\n",
    "\n",
    "through [Load optanalysis results](#Load-optanalysis-results) and then ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRefRes = results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or: Load results from _REPORT_ file\n",
    "\n",
    "through [2. Or: Load existing industrialised report](#2.-Or%3A-Load-existing-industrialised-report) or\n",
    "[3. Or: Generate industrialised filter & sort report from existing results](#3.-Or%3A-Generate-industrialised-filter-%26-sort-report-from-existing-results) and then ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(', '.join(ddfIndusRep.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRefRes = ddfIndusRep['Détails']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRefRes.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Target results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or: Load results from _RESULTS_ file\n",
    "\n",
    "through [Load optanalysis results](#Load-optanalysis-results) and then ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfActRes = results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or: Load results from _REPORT_ file\n",
    "\n",
    "through [2. Or: Load existing industrialised report](#2.-Or%3A-Load-existing-industrialised-report) or\n",
    "[3. Or: Generate industrialised filter & sort report from existing results](#3.-Or%3A-Generate-industrialised-filter-%26-sort-report-from-existing-results) and then ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(', '.join(ddfIndusRep.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfActRes = ddfIndusRep['Détails']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfActRes.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Truncation groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check strict equality\n",
    "indexCols = ['Analyse']\n",
    "compCols = ['Echant', 'Groupe Tronc Gche', 'Groupe Tronc Drte']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert dfRefRes.set_index(indexCols).sort_index()[compCols].compare(dfActRes.set_index(indexCols).sort_index()[compCols]).empty"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Filter and sort keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfActRes.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check strict equality\n",
    "indexCols = ['Analyse']\n",
    "compCols = [col for col in dfActRes.columns if col.startswith('Ordre ')]\n",
    "\n",
    "', '.join(compCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert dfRefRes.set_index(indexCols).sort_index()[compCols].compare(dfActRes.set_index(indexCols).sort_index()[compCols]).empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MCDSTruncOptAnalysisResultsSet quality indicators progress\n",
    "\n",
    "General target: Have the industrialised automated filter and sort report produce the same \"winner result per sample\", or a better one, when compared to the \"final automated [protopype below](#D%C3%A9veloppement-%3A-Filtrage-et-tri-automatis%C3%A9-des-r%C3%A9sultats-d'optanalyses) + manual selection\" reports donnees/*/ACDC2019-[Nat|Pap|NatPap]-OptAnalyses-raptousech.ods\n",
    "\n",
    "Here, we try and compare filter and sort results from :\n",
    "* reference = historical [protopype below](#D%C3%A9veloppement-%3A-Filtrage-et-tri-automatis%C3%A9-des-r%C3%A9sultats-d'optanalyses) + manual selection\" reports donnees/*/ACDC2019-[Nat|Pap|NatPap]-OptAnalyses-raptousech.ods\n",
    "* actual = a newer auto-generated report through the industrialised filter and sort report system.\n",
    "\n",
    "WARNING: No more really useful as most filtered and sorted results now select mostly different analyses than in the historical case."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Select target filter and sort method for prototype report\n",
    "\n",
    "WARNING: First run [1. Or: Load early 2021 prototype report](#1.-Or%3A-Load-early-2021-prototype-report) to load the full report."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select target filter & sort method sub-report\n",
    "#refExCMeth = 'codexec'\n",
    "refTgtMeth = 'ckcvqual925d12'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRefRep = ddfProtoRep[protoSheetPrefix + refTgtMeth]\n",
    "\n",
    "logger.info('Référence (proto) : {}x{} lignes x colonnes.'.format(len(dfRefRep), len(dfRefRep.columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract useful columns in the right order\n",
    "dfRefRep = dfRefRep[['Echant', 'Espèce', 'Passage', 'Adulte', 'Durée', 'Analyse',\n",
    "                     'FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod',\n",
    "                     'NbTot Pars', 'NObs', 'NTot Obs', 'Taux Obs', 'CodEx', \n",
    "                     'Delta AIC', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P',\n",
    "                     'CoefVar Densité', 'Sélection finale',\n",
    "                     'Pré-sélection Qual Equi 1', 'Qual Equi 1',\n",
    "                     'Qual Chi2+', 'Qual KS+', 'Qual DCv+',\n",
    "                     'Densité', 'Min Densité', 'Max Densité']].copy()\n",
    "\n",
    "# Add source column\n",
    "dfRefRep['Ref'] = True\n",
    "\n",
    "# Prepare comparison of quality indicators specific columns\n",
    "dfRefRep.rename(columns={col: col + ' REF' for col in ['Pré-sélection Qual Equi 1', 'Qual Equi 1',\n",
    "                                                       'Qual Chi2+', 'Qual DCv+', 'Qual KS+']},\n",
    "                inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some checks before going on.\n",
    "assert dfRefRep.Analyse.nunique() == len(dfRefRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRefRep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Select target filter and sort method for industrialised report\n",
    "\n",
    "WARNING: First run [2. Or: Load existing industrialised report](#2.-Or%3A-Load-existing-industrialised-report) or\n",
    "[3. Or: Generate industrialised filter & sort report from existing results](#3.-Or%3A-Generate-industrialised-filter-%26-sort-report-from-existing-results) to load the full report."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "', '.join(ddfIndusRep.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select target filter & sort method sub-report\n",
    "#actTgtMeth = 'ExCode'\n",
    "\n",
    "#actTgtMeth = 'AicCKCvQua-r925d12' # Old industrialised naming before 2021-10-15\n",
    "\n",
    "#actTgtMeth = 'ExAicMQua-r925d12'  # Intermediate industrialised naming between 2021-10-15 and 2021-11-05\n",
    "\n",
    "actTgtMeth = 'ExAicMQua-r925m6q3d12' # New industrialised naming for same method/scheme after 2021-11-05\n",
    "#actTgtMeth = 'ExAicMQua-r925m6q2d12'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfActRep = ddfIndusRep[indusSheetPrefix + actTgtMeth]\n",
    "\n",
    "logger.info('Cible (indus) : {}x{} lignes x colonnes.'.format(len(dfActRep), len(dfActRep.columns)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfActRep.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract useful columns in the right order\n",
    "nParsCol = 'NbTot Pars' if 'NbTot Pars' in dfActRep.columns else 'NbPars SérAjust'\n",
    "codExCol = ['CodEx'] if 'CodEx' in dfActRep.columns else []\n",
    "dfActRep = dfActRep[['Echant', 'Espèce', 'Passage', 'Adulte', 'Durée', 'Analyse',\n",
    "                     'FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod',\n",
    "                     nParsCol, 'NObs', 'NTot Obs', 'Taux Obs'] + codExCol \\\n",
    "                     + ['Delta AIC', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P',\n",
    "                        'CoefVar Densité',\n",
    "                        'Pré-sélection Qual Equi 1', 'Qual Equi 1',\n",
    "                        'Pré-sélection Qual Equi 2', 'Qual Equi 2',\n",
    "                        'Pré-sélection Qual Equi 3', 'Qual Equi 3',\n",
    "                        'Qual Chi2+', 'Qual KS+', 'Qual DCv+',\n",
    "                        'Densité', 'Min Densité', 'Max Densité']].copy()\n",
    "\n",
    "# Add source column\n",
    "dfActRep['Act'] = True\n",
    "\n",
    "# Prepare comparison of quality indicators specific columns\n",
    "dfActRep.rename(columns={col: col + ' ACT' for col in ['Pré-sélection Qual Equi 1', 'Qual Equi 1',\n",
    "                                                       'Pré-sélection Qual Equi 2', 'Qual Equi 2',\n",
    "                                                       'Pré-sélection Qual Equi 3', 'Qual Equi 3',\n",
    "                                                       'Qual Chi2+', 'Qual DCv+', 'Qual KS+']},\n",
    "                inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some checks before going on.\n",
    "assert dfActRep.Analyse.nunique() == len(dfActRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfActRep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Compare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Visual comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfRefRep), len(dfActRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Merge reference and actual reports\n",
    "actJoinCols = [col + ' ACT' for col in ['Pré-sélection Qual Equi 1', 'Qual Equi 1',\n",
    "                                        'Pré-sélection Qual Equi 2', 'Qual Equi 2',\n",
    "                                        'Pré-sélection Qual Equi 3', 'Qual Equi 3',\n",
    "                                        'Qual Chi2+', 'Qual DCv+', 'Qual KS+']] + ['Act']\n",
    "dfComp = dfRefRep.set_index('Analyse').join(dfActRep.set_index('Analyse')[actJoinCols])\n",
    "\n",
    "dfComp = pd.concat([dfComp, dfActRep[~dfActRep.Analyse.isin(dfComp.index)].set_index('Analyse')])\n",
    "\n",
    "dfComp.reset_index(inplace=True)\n",
    "dfComp.rename(columns=dict(index='Analyse'), inplace=True)\n",
    "\n",
    "dfComp.sort_values(by=['Echant', 'TrGche', 'TrDrte', 'Qual Equi 1 REF'], ascending=True, na_position='first', inplace=True)\n",
    "\n",
    "compFileName = f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-quality-progress.xlsx'\n",
    "dfComp.to_excel(tmpDir / compFileName, sheet_name=actTgtMeth, index=False)\n",
    "\n",
    "dfComp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some checks before going on.\n",
    "assert dfComp.Analyse.nunique() == len(dfComp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Simple results quality progress measurement\n",
    "\n",
    "(as developpement goes on)\n",
    "\n",
    "Warning: Only useful when most filtered and sorted results come from the same analyses ... BUT, this is less the case after 2021-10-10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare reference final = manual selection, reference auto pre-selection, and actual auto pre-selection\n",
    "nulComp = pd.Series(dict(goodRefSel1=np.nan, goodActSel=np.nan, betterActSel=np.nan, sharedRefAct=np.nan))\n",
    "def compare2Ref(dfGrp, column='Pré-sélection Qual Equi 3 ACT'):\n",
    "    \n",
    "    # Extract reference final = manual selection\n",
    "    sFinSel = dfGrp[dfGrp['Sélection finale'].notnull()]\n",
    "    if sFinSel.empty:\n",
    "        return nulComp\n",
    "    sFinSel = sFinSel.iloc[0]\n",
    "    \n",
    "    # Extract reference auto pre-selection (on Qual Equi 1)\n",
    "    sRefPSel = dfGrp[dfGrp['Pré-sélection Qual Equi 1 REF'] == 1]\n",
    "    if sRefPSel.empty:\n",
    "        return nulComp\n",
    "    sRefPSel = sRefPSel.iloc[0]\n",
    "    \n",
    "    # Extract actual auto pre-selection (on specified column)\n",
    "    sActPSel = dfGrp[dfGrp[column] == 1]\n",
    "    if sActPSel.empty:\n",
    "        return nulComp\n",
    "    sActPSel = sActPSel.iloc[0]\n",
    "    \n",
    "    # Report: Compare ref final to ref, ref final to actual ; and check if \n",
    "    return pd.Series(dict(goodRefSel1=1 if (sFinSel.Analyse == sRefPSel.Analyse) else 0,\n",
    "                          goodActSel=1 if (sFinSel.Analyse == sActPSel.Analyse) else 0,\n",
    "                          betterActSel=1 if (sFinSel['Pré-sélection Qual Equi 1 REF'] >= sFinSel[column]) else 0,\n",
    "                          sharedRefAct=(dfGrp['Ref'] & dfGrp['Act']).sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfDiag2 = dfComp.groupby('Echant').apply(compare2Ref, column='Pré-sélection Qual Equi 2 ACT')\n",
    "dfDiag2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(dfComp), 'analyses:', len(dfDiag2), 'samples:', dfDiag2.sum().to_dict())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2021-10-10\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal1\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal1\n",
    "\n",
    "* Progress history of len(dfDiag2), dfDiag2.sum():\n",
    "  * 12:00 => 60 samples: {goodRefSel1: 31, goodActSel: 34, betterActSel: 48}\n",
    "  * 18:00 => 60 samples: {goodRefSel1: 31, goodActSel: 33, betterActSel: 47}\n",
    "  * 19:20 => 60 samples: {goodRefSel1: 31, goodActSel: 32, betterActSel: 46}\n",
    "\n",
    "* Visual checks OK on a 9 sample subset of ACDC 2019 Nat:\n",
    "  4 samples with > 120/200 sightings, 4 samples with 35-80 sightings, and 1 sample with 25-30 sightings\n",
    "\n",
    "#### 2021-11-01 CLGrpOrdClTrQuaBal1 after MCDSAnalyserResultsSet fixes of the day\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal1\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal1\n",
    "\n",
    "* len(dfComp), len(dfDiag2), dfDiag2.sum().to_dict():\n",
    "  925 analyses: {sharedRefAct: 454}, 60 samples, {goodRefSel1: 31, goodActSel: 30, betterActSel: 39}\n",
    "  \n",
    "#### 2021-11-02 09:00 Mix of CLGrpOrdClTrQuaBal1, 2 & 3 ... not very interesting\n",
    "\n",
    "After fixing buggy MCDSAnalyser.filterDichotScheme (now _indexOfWorstOneCriterion)\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = CLGrpOrdClTrQuaBal2, then CLGrpOrdClTrQuaBal3\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal1 <= QuaBal1 and not 2/3 :-(\n",
    "\n",
    "* len(dfComp), len(dfDiag2), dfDiag2.sum().to_dict():\n",
    "  - CLGrpOrdClTrQuaBal3 => 969 analyses: {sharedRefAct: 410}, 60 samples: {goodRefSel1: 31, goodActSel: 28, betterActSel: 39}\n",
    "  - CLGrpOrdClTrQuaBal2 => 965 analyses: {sharedRefAct: 414}, 60 samples: {goodRefSel1: 31, goodActSel: 28, betterActSel: 39}\n",
    " \n",
    "#### 2021-11-02 10:20 CLGrpOrdClTrQuaBal2 only\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal2\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal2 <= QuaBal2 :-)\n",
    "\n",
    "* len(dfComp), len(dfDiag2), dfDiag2.sum().to_dict():\n",
    "  966 analyses: {sharedRefAct: 409}, 60 samples: {goodRefSel1: 31, goodActSel: 27, betterActSel: 39}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfDiag3 = dfComp.groupby('Echant').apply(compare2Ref, column='Pré-sélection Qual Equi 3 ACT')\n",
    "dfDiag3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(dfComp), 'analyses:', len(dfDiag3), 'samples:', dfDiag3.sum().to_dict())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2021-10-10\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal1\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal1\n",
    "\n",
    "* Progress history of len(dfDiag3), dfDiag3.sum() :\n",
    "  - 12:00 => (60, {rePSel: 31, sel: 33, better: 47})\n",
    "  - 18:00 => (60, {rePSel: 31, sel: 30, better: 41})\n",
    "  - 19:20 => (60, {rePSel: 31, sel: 30, better: 40})\n",
    "\n",
    "* Visual checks OK on a 9 sample subset of ACDC 2019 Nat:\n",
    "  4 samples with > 120/200 sightings, 4 samples with 35-80 sightings, and 1 sample with 25-30 sightings\n",
    " \n",
    "\n",
    "#### 2021-11-01 CLGrpOrdClTrQuaBal1 after MCDSAnalyserResultsSet fixes of the day\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal1\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal1\n",
    "\n",
    "* len(dfComp), len(dfDiag3), dfDiag3.sum().to_dict():\n",
    "  925 analyses: {sharedRefAct: 454}, 60 samples, {goodRefSel1: 31, goodActSel: 28, betterActSel: 33}\n",
    "\n",
    "#### 2021-11-02 09:00 Mix of CLGrpOrdClTrQuaBal1, 2 & 3 ... not very interesting\n",
    "\n",
    "After fixing buggy MCDSAnalyser.filterDichotScheme (now _indexOfWorstOneCriterion)\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal2, then R.CLGrpOrdClTrQuaBal3\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal1 <= QuaBal1 and not 2/3 :-(\n",
    "\n",
    "* len(dfComp), len(dfDiag3), dfDiag3.sum().to_dict():\n",
    "  - CLGrpOrdClTrQuaBal3 => 969 analyses: {sharedRefAct: 410}, 60 samples, {goodRefSel1: 31, goodActSel: 26, betterActSel: 33}\n",
    "  - CLGrpOrdClTrQuaBal2 => 965 analyses: {sharedRefAct: 414}, 60 samples: {goodRefSel1: 31, goodActSel: 26, betterActSel: 33}\n",
    "  \n",
    "#### 2021-11-02 10:20 CLGrpOrdClTrQuaBal3 only\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal3\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal3 <= QuaBal3 :-)\n",
    "\n",
    "* len(dfComp), len(dfDiag3), dfDiag3.sum().to_dict():\n",
    "  970 analyses: {sharedRefAct: 405}, 60 samples: {goodRefSel1: 31, goodActSel: 25, betterActSel: 32}\n",
    "  \n",
    "#### 2021-11-03 19:00 CLGrpOrdClTrQuaBal3 only\n",
    "\n",
    "After fixing final sort column (QuaOrd => Qua !) in MCDSAnalysisResultsSet.filterSortOnExCAicMulQua/filterSortOnExCode\n",
    "=> better !\n",
    "\n",
    "* Actual Fil/Sor params:\n",
    "  - whichFinalQua = R.CLGrpOrdClTrQuaBal3\n",
    "  - whichBestQua = R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv, R.CLGrpOrdClTrQuaBal3 <= QuaBal3 :-)\n",
    "\n",
    "* len(dfComp), len(dfDiag3), dfDiag3.sum().to_dict():\n",
    "  859 analyses: {sharedRefAct: 516}, 60 samples: {goodRefSel1: 31, goodActSel: 30, betterActSel: 40}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## N. Development of base functions for quality indicators computation\n",
    "\n",
    "History (on ACDC 2019 data):\n",
    "* March 2021\n",
    "  * Balanced quality 1 : normNTotPars(a=0.2, b=0.6, c=2) & normCVDens(a=12) => ACDC2019-Resultats.2103.ods\n",
    "* August 2021\n",
    "  * Balanced quality 2 : normNTotPars(a=0.2, b=0.8, c=1) & normCVDens(a=16) => same final filtering on AicCKCvQua-r925d12\n",
    "  * Balanced quality 3 : normNTotPars(a=0.3, b=0.7, c=1) & normCVDens(a=20) => ~same final filtering on AicCKCvQua-r925d12\n",
    "* October 2021\n",
    "  * Balanced quality 2&3 : same as August 2021 but normNTotPars => normNKeyPars + normKeyFn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import plotly as ply\n",
    "import plotly.graph_objs as plygo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normNTotPars(value, a=0.2, b=0.6, c=2):  #, d=1):\n",
    "    #return 1 / (a * value + b)  # Trop pénalisant: a=0.2, b=1\n",
    "    return 1 / (a * max(c, value) + b)  # Mieux: a=0.2, b=0.6, c=2 / a=0.2, b=0.8, c=1\n",
    "    #return 1 / (a * max(c, value)**d + b)  # Idem si d=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ax = np.linspace(0, 10, num=11)\n",
    "\n",
    "ySchemes = dict(bq1mar21=dict(a=0.2, b=0.6, c=2), # Balanced quality 1  March 2021\n",
    "                bq2aug21=dict(a=0.2, b=0.8, c=1), # Balanced quality 2 August 2021 : no change in final filsorting\n",
    "                bq3aug21=dict(a=0.3, b=0.7, c=1)) # Balanced quality 3 August 2021 : ~idem\n",
    "\n",
    "plygo.Figure(data=[plygo.Scatter(x=ax, name=name + ': ' + ', '.join('{}={}'.format(k, v) for k, v in sch.items()),\n",
    "                                 y=[normNTotPars(x, **sch) for x in ax])\n",
    "                   for name, sch in ySchemes.items()],\n",
    "             layout=dict(title='normNTotPars', height=320, width=768,\n",
    "                         margin=plygo.layout.Margin(l=40, r=40, b=40, t=40, pad=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normNAdjPars(value, a=0.1): #, b=0.6):\n",
    "    #return 1 / (a * value + b)  # Pénalise trop à faible x, pas assez après\n",
    "    return math.exp(-a * value ** 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ax = np.linspace(0, 7, num=8)\n",
    "\n",
    "ySchemes = dict(tbq2oct21=dict(a=0.10), # Balanced quality 2 October 2021\n",
    "                bq2oct21=dict(a=0.15), # Balanced quality 2 October 2021\n",
    "                bq3oct21=dict(a=0.20), # Balanced quality 2 October 2021\n",
    "                tbq3oct21=dict(a=0.25)) # Balanced quality 3 October 2021\n",
    "\n",
    "plygo.Figure(data=[plygo.Scatter(x=ax, name=name + ': ' + ', '.join('{}={}'.format(k, v) for k, v in sch.items()),\n",
    "                                 y=[normNAdjPars(x, **sch) for x in ax])\n",
    "                   for name, sch in ySchemes.items()],\n",
    "             layout=dict(title='normNAdjPars', height=320, width=768,\n",
    "                         margin=plygo.layout.Margin(l=40, r=40, b=40, t=40, pad=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normCVDens(value, a=12, b=2):\n",
    "    #return max(0, 1 - a * value) # Pas très pénalisant: a=1\n",
    "    return math.exp(-a * value ** b) # Mieux : déjà ~0.33 à 30% (a=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = np.linspace(0.0, 1.0, num=11)\n",
    "ySchemes = dict(bq1mar21=dict(a=12, b=2), # Balanced quality 1 March  2021\n",
    "                tbq2oct21=dict(a=16, b=2), # Test Balanced quality 2 October 2021\n",
    "                bq2oct21=dict(a=20, b=2), # Balanced quality 2 October 2021\n",
    "                #tbq3oct21=dict(a=25, b=2), # Test Balanced quality 3 October 2021\n",
    "                tbq3oct21b=dict(a=55, b=2.6), # Test Balanced quality 3 October 2021\n",
    "                bq3oct21c=dict(a=63, b=2.8)) # Test Balanced quality 3 October 2021\n",
    "plygo.Figure(data=[plygo.Scatter(x=ax, name=name + ': ' + ', '.join('{}={}'.format(k, v) for k, v in sch.items()),\n",
    "                                 y=[normCVDens(x, **sch) for x in ax])\n",
    "                   for name, sch in ySchemes.items()],\n",
    "             layout=dict(title='normCVDens', height=320, width=768,\n",
    "                         margin=plygo.layout.Margin(l=40, r=40, b=40, t=40, pad=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Non-regression: MCDS(TruncOpt)AnalysisResultsSet\n",
    "\n",
    "* Non-regression tests between [protopype below](#D%C3%A9veloppement-%3A-Filtrage-et-tri-automatis%C3%A9-des-r%C3%A9sultats-d'optanalyses) and \"industrialised\" version\n",
    "  (Ehrrr ... well ... between the buggy notebook prototype and the industrialised-derived version with same bugs :-(\n",
    "* Quality tests for \"industrialised\" version (TODO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ads.logger('ads.dat', level=ads.INFO, reset=True)\n",
    "_ = ads.logger('ads.anr', level=ads.DEBUG3, reset=True)\n",
    "_ = ads.logger('ads.onr', level=ads.DEBUG3, reset=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The industrialised-derived optanalyser results set for reproducing the buggy prototype version\n",
    "class PrototypeConformResultsSet(ads.MCDSTruncOptanalysisResultsSet):\n",
    "\n",
    "    def __init__(self, miCustomCols=None, dfCustomColTrans=None, miSampleCols=None, sampleIndCol=None,\n",
    "                       sortCols=[], sortAscend=[], distanceUnit='Meter', areaUnit='Hectare',\n",
    "                       surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                       ldTruncIntrvSpecs=[dict(col='left', minDist=5.0, maxLen=5.0),\n",
    "                                          dict(col='right', minDist=25.0, maxLen=25.0)],\n",
    "                       truncIntrvEpsilon=1e-6):\n",
    "        \n",
    "        \"\"\"\n",
    "        \"\"\"\n",
    "        super().__init__(miCustomCols=miCustomCols, dfCustomColTrans=dfCustomColTrans,\n",
    "                         miSampleCols=miSampleCols, sampleIndCol=sampleIndCol,\n",
    "                         sortCols=sortCols, sortAscend=sortAscend, distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                         surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                         ldTruncIntrvSpecs=ldTruncIntrvSpecs,\n",
    "                         truncIntrvEpsilon=truncIntrvEpsilon)\n",
    "\n",
    "    @classmethod\n",
    "    def _combinedQualityMoreChi2(cls, sRes):  # Prototype bug: (x*y*...*z)^(1/8) ; should be: x*y*...*z^(1/8)\n",
    "        return sRes[[cls.CLChi2, cls.CLChi2, cls.CLKS, cls.CLCvMUw, cls.CLCvMCw]].prod() \\\n",
    "               * cls._normNObs(sRes) * cls._normNTotPars(sRes, a=0.2, b=0.6) \\\n",
    "               * cls._normCVDens(sRes, a=12) ** (1.0/8)\n",
    "\n",
    "    @classmethod\n",
    "    def _combinedQualityMoreKS(cls, sRes):  # Prototype bug: idem\n",
    "        return sRes[[cls.CLChi2, cls.CLKS, cls.CLKS, cls.CLCvMUw, cls.CLCvMCw]].prod() \\\n",
    "               * cls._normNObs(sRes) * cls._normNTotPars(sRes, a=0.2, b=0.6) \\\n",
    "               * cls._normCVDens(sRes, a=12) ** (1.0/8)\n",
    "\n",
    "    @classmethod\n",
    "    def _combinedQualityMoreDCv(cls, sRes):  # Prototype bug: idem\n",
    "        return sRes[[cls.CLChi2, cls.CLKS, cls.CLCvMUw, cls.CLCvMCw]].prod() \\\n",
    "               * cls._normNObs(sRes) * cls._normNTotPars(sRes, a=0.2, b=0.6) \\\n",
    "               * (cls._normCVDens(sRes, a=12) ** 2) ** (1.0/8)\n",
    "\n",
    "    def _postComputeQualityIndicators(self):\n",
    "        \n",
    "        logger.debug('Post-computing Quality Indicators')\n",
    "\n",
    "        self._dfData[self.CLSightRate] = 100 * self._dfData.apply(self._normNObs, axis='columns') # [0,1] => %\n",
    "\n",
    "        # Prepare data for computations\n",
    "        miCompCols = [cls.CLNObs, cls.CLNTotObs, cls.CLNTotPars, \n",
    "                      cls.CLChi2, cls.CLKS, cls.CLCvMUw, cls.CLCvMCw, cls.CLDCv]\n",
    "        dfCompData = self._dfData[miCompCols].copy()\n",
    "\n",
    "        logger.debug1('* Balanced quality 1')\n",
    "        self._dfData[self.CLCmbQuaBal1] = dfCompData.apply(self._combinedQualityBalanced1, axis='columns')\n",
    "\n",
    "        logger.debug1('* Balanced quality 2')\n",
    "        self._dfData[self.CLCmbQuaBal2] = dfCompData.apply(self._combinedQualityBalanced2, axis='columns')\n",
    "\n",
    "        logger.debug1('* Balanced quality 3')\n",
    "        self._dfData[self.CLCmbQuaBal3] = dfCompData.apply(self._combinedQualityBalanced3, axis='columns')\n",
    "\n",
    "        logger.debug1('* Balanced quality Chi2+')\n",
    "        self._dfData[self.CLCmbQuaChi2] = dfCompData.apply(self._combinedQualityMoreChi2, axis='columns')\n",
    "\n",
    "        logger.debug1('* Balanced quality KS+')\n",
    "        self._dfData[self.CLCmbQuaKS]   = dfCompData.apply(self._combinedQualityMoreKS, axis='columns')\n",
    "\n",
    "        logger.debug1('* Balanced quality DCv+')\n",
    "        self._dfData[self.CLCmbQuaDCv]  = dfCompData.apply(self._combinedQualityMoreDCv, axis='columns')\n",
    "\n",
    "# And the industrialised-derived optanalyser for instanciating it easily\n",
    "class PrototypeConformOptanalyser(ads.MCDSTruncationOptanalyser):\n",
    "\n",
    "    def __init__(self, dfMonoCatObs, dfTransects=None, effortConstVal=1, dSurveyArea=dict(), \n",
    "                 transectPlaceCols=['Transect'], passIdCol='Pass', effortCol='Effort',\n",
    "                 sampleSelCols=['Species', 'Pass', 'Adult', 'Duration'], \n",
    "                 sampleDecCols=['Effort', 'Distance'], sampleDistCol='Distance', anlysSpecCustCols=[],\n",
    "                 abbrevCol='AnlysAbbrev', abbrevBuilder=None, anlysIndCol='AnlysNum', sampleIndCol='SampleNum',\n",
    "                 distanceUnit='Meter', areaUnit='Hectare',\n",
    "                 surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                 resultsHeadCols=dict(before=['AnlysNum', 'SampleNum'], after=['AnlysAbbrev'], \n",
    "                                      sample=['Species', 'Pass', 'Adult', 'Duration']),\n",
    "                 ldTruncIntrvSpecs=[dict(col='left', minDist=5.0, maxLen=5.0),\n",
    "                                    dict(col='right', minDist=25.0, maxLen=25.0)], truncIntrvEpsilon=1e-6,\n",
    "                 workDir='.', runMethod='subprocess.run', runTimeOut=300, logData=False,\n",
    "                 logAnlysProgressEvery=50, logOptimProgressEvery=5, backupOptimEvery=50, autoClean=True,\n",
    "                 defEstimKeyFn=ads.MCDSEngine.EstKeyFnDef, defEstimAdjustFn=ads.MCDSEngine.EstAdjustFnDef,\n",
    "                 defEstimCriterion=ads.MCDSEngine.EstCriterionDef, defCVInterval=ads.MCDSEngine.EstCVIntervalDef,\n",
    "                 defMinDist=ads.MCDSEngine.DistMinDef, defMaxDist=ads.MCDSEngine.DistMaxDef, \n",
    "                 defFitDistCuts=ads.MCDSEngine.DistFitCutsDef, defDiscrDistCuts=ads.MCDSEngine.DistDiscrCutsDef,\n",
    "                 defExpr2Optimise='chi2', defMinimiseExpr=False,\n",
    "                 defOutliersMethod='tucquant', defOutliersQuantCutPct=5,\n",
    "                 defFitDistCutsFctr=dict(min=2/3, max=3/2),\n",
    "                 defDiscrDistCutsFctr=dict(min=1/3, max=1),\n",
    "                 defSubmitTimes=1, defSubmitOnlyBest=None, dDefSubmitOtherParams=dict(),\n",
    "                 dDefOptimCoreParams=dict(core='zoopt', maxIters=100, termExprValue=None,\n",
    "                                          algorithm='racos', maxRetries=0)):\n",
    "\n",
    "        \n",
    "        super().__init__(dfMonoCatObs, dfTransects=dfTransects, effortConstVal=effortConstVal, dSurveyArea=dSurveyArea, \n",
    "                         transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                         sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                         sampleDistCol=sampleDistCol, anlysSpecCustCols=anlysSpecCustCols,\n",
    "                         abbrevCol=abbrevCol, abbrevBuilder=abbrevBuilder,\n",
    "                         anlysIndCol=anlysIndCol, sampleIndCol=sampleIndCol,\n",
    "                         distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                         surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                         resultsHeadCols=resultsHeadCols,\n",
    "                         ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon,\n",
    "                         workDir=workDir, runMethod=runMethod, runTimeOut=runTimeOut, logData=logData,\n",
    "                         logAnlysProgressEvery=logAnlysProgressEvery, logOptimProgressEvery=logOptimProgressEvery,\n",
    "                         backupOptimEvery=backupOptimEvery, autoClean=autoClean,\n",
    "                         defEstimKeyFn=defEstimKeyFn, defEstimAdjustFn=defEstimAdjustFn,\n",
    "                         defEstimCriterion=defEstimCriterion, defCVInterval=defCVInterval,\n",
    "                         defMinDist=defMinDist, defMaxDist=defMaxDist, \n",
    "                         defFitDistCuts=defFitDistCuts, defDiscrDistCuts=defDiscrDistCuts,\n",
    "                         defExpr2Optimise=defExpr2Optimise, defMinimiseExpr=defMinimiseExpr,\n",
    "                         defOutliersMethod=defOutliersMethod, defOutliersQuantCutPct=defOutliersQuantCutPct,\n",
    "                         defFitDistCutsFctr=defFitDistCutsFctr, defDiscrDistCutsFctr=defDiscrDistCutsFctr,\n",
    "                         defSubmitTimes=defSubmitTimes, defSubmitOnlyBest=defSubmitOnlyBest,\n",
    "                         dDefSubmitOtherParams=dDefSubmitOtherParams, dDefOptimCoreParams=dDefOptimCoreParams)\n",
    "\n",
    "    def setupResults(self):\n",
    "    \n",
    "        \"\"\"Build an empty results objects.\n",
    "        \"\"\"\n",
    "\n",
    "        miCustCols, dfCustColTrans, miSampCols, sampIndMCol, sortCols, sortAscend = \\\n",
    "            self.prepareResultsColumns()\n",
    "\n",
    "        return PrototypeConformResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                          miSampleCols=miSampCols, sampleIndCol=sampIndMCol,\n",
    "                                          sortCols=sortCols, sortAscend=sortAscend,\n",
    "                                          distanceUnit=self.distanceUnit, areaUnit=self.areaUnit,\n",
    "                                          surveyType=self.surveyType, distanceType=self.distanceType,\n",
    "                                          clustering=self.clustering,\n",
    "                                          ldTruncIntrvSpecs=self.ldTruncIntrvSpecs,\n",
    "                                          truncIntrvEpsilon=self.truncIntrvEpsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load results to postCompute from file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dossier = pl.Path('../../perso/donnees/acdc')\n",
    "\n",
    "nomEtude = 'ACDC2019'\n",
    "sousEtude = '-Nat'\n",
    "\n",
    "varEtude = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colEspece = 'Espèce'\n",
    "colPassage = 'Passage'\n",
    "colDistance = 'Distance'\n",
    "\n",
    "groupage = False\n",
    "effortConst = 1 # Valeur d'effort constante = 1 par passage sur chaque point."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chargement des données\n",
    "fpn = dossier / f'{nomEtude}{sousEtude}-ObsIndivDist.xlsx'\n",
    "with pd.ExcelFile(fpn) as xlsFile:\n",
    "    dfObsCatIndiv = pd.read_excel(xlsFile, sheet_name='Donnees')\n",
    "    dfTransects = pd.read_excel(xlsFile, sheet_name='Inventaires')\n",
    "\n",
    "print(dict(etude=nomEtude+sousEtude, donnees=len(dfObsCatIndiv), inventaires=len(dfTransects)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Description des données\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = colPassage\n",
    "\n",
    "assert 'effortCol' not in dir() or effortCol == 'Effort'  # In rare cases, needs to be defined before here, but the same way !\n",
    "effortCol = 'Effort'\n",
    "\n",
    "colsSpeSelEchant = ['Adulte', 'Durée']  # Colonnes de sélection des échantillons : en plus de Espèce et Passage. \n",
    "sampleDistCol = colDistance\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'Echant'\n",
    "sampleSelCols = [colEspece, passIdCol] + colsSpeSelEchant\n",
    "\n",
    "#sampleAbbrevCol = 'Abrev. Echant'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Compléments pour les analyses.\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Sq. Kilometer'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "\n",
    "dZoneEtude = dict(Zone='ACDC', Surface=24) # km2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Compléments pour les optanalyses.\n",
    "anlysIndCol = 'Analyse'\n",
    "anlysAbbrevCol = 'Abrev. Analyse'\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paramètres pour le groupage des troncatures (indicateurs qualités)\n",
    "ldTruncIntrvSpecs = [dict(col='left', minDist=5.0, maxLen=5.0),  dict(col='right', minDist=25.0, maxLen=25.0)]\n",
    "truncIntrvEpsilon = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An optanalyser object knowns how to build an empty results object ...\n",
    "optanlr = \\\n",
    "    PrototypeConformOptanalyser(dfObsCatIndiv, dfTransects=dfTransects,\n",
    "                                effortConstVal=effortConst, dSurveyArea=dZoneEtude, \n",
    "                                transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                                sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                                abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                                anlysIndCol=anlysIndCol, sampleIndCol=sampleNumCol,\n",
    "                                distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                                surveyType=surveyType, distanceType=distanceType, clustering=groupage,\n",
    "                                ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon,\n",
    "                                resultsHeadCols=dict(before=[anlysIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                                     after=anlysParamCols + [anlysAbbrevCol]))\n",
    "\n",
    "results = optanlr.setupResults()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resFileName = f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-resultats.xlsx'\n",
    "\n",
    "resFolders = [fn.name for fn in dossier.glob('[0-9]'*6+'-'+'[0-9]'*4) if (fn / resFileName).is_file()]\n",
    "\n",
    "print('Résultats historiques disponibles:', ', '.join(resFolders))\n",
    "\n",
    "workDir = dossier / resFolders[0]  # <=== Choisir le dossier de résultats ici.\n",
    "\n",
    "updatedResFileNameExists = (pl.Path('tmp') / resFileName).is_file()\n",
    "if not updatedResFileNameExists:\n",
    "\n",
    "    resFileName = workDir / resFileName\n",
    "\n",
    "    print(f'Fichier choisi : {resFileName.as_posix()}')\n",
    "    \n",
    "else:\n",
    "    \n",
    "    resFileName = pl.Path('tmp') / resFileName\n",
    "\n",
    "    print(f'... mais résultats mis à jour aussi: {resFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Load results from file\n",
    "print('Lecture du fichier choisi:', resFileName.as_posix(), '...')\n",
    "\n",
    "results.fromFile(resFileName, postComputed=updatedResFileNameExists)\n",
    "\n",
    "print('... terminé.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not updatedResFileNameExists:\n",
    "    \n",
    "    # Add sample stats a posteriori (these stats had not been implemented when the historical results were saved to disk)\n",
    "    dfSampleStats = pd.read_excel(dossier / f'{nomEtude}{sousEtude}-StatsEchantillons.xlsx')\n",
    "    dfSampleStats.rename(columns={'NTot Obs': 'NTot Obs0'}, inplace=True)\n",
    "    dfSampleStats.insert(dfSampleStats.columns.to_list().index('Distance Min'), 'NTot Obs', dfSampleStats['NTot Obs0'])\n",
    "    dfSampleStats.drop(columns=['NTot Obs0'], inplace=True)\n",
    "\n",
    "    miSampleCols = pd.MultiIndex.from_tuples([('header (sample)', colEspece, 'Value'),\n",
    "                                              ('header (sample)', colPassage, 'Value'),\n",
    "                                              ('header (sample)', colsSpeSelEchant[0], 'Value'),\n",
    "                                              ('header (sample)', colsSpeSelEchant[1], 'Value')])\n",
    "    dfSampleStats.columns = miSampleCols.append(ads.MCDSEngine.MIStatSampCols)\n",
    "\n",
    "    results.dfData = results._dfData.join(dfSampleStats.set_index(miSampleCols.to_list()), on=miSampleCols.to_list())\n",
    "    \n",
    "    print(len(results._dfData))\n",
    "    \n",
    "    print('Ecriture du fichier mis à jour:', (pl.Path('tmp') / resFileName.name).as_posix(), '...')\n",
    "    results.toExcel(pl.Path('tmp') / resFileName.name)\n",
    "    print('... terminé.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results._dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Trigger pos-computations now !\n",
    "dfRes = results.dfTransData('fr')\n",
    "dfRes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "fpn = tmpDir / resFileName.name.replace('resultats', 'resultats-postcalc')\n",
    "\n",
    "print('Ecriture du tableau de résultats post-calculé :',fpn.as_posix(), '...')\n",
    "\n",
    "dfRes.to_excel(fpn)\n",
    "\n",
    "print('... terminé.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load prototype enriched simple results report "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repFileName = workDir / f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-rapenrich.xlsx'\n",
    "\n",
    "print(f'Fichier choisi : {repFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfRefRep = pd.read_excel(repFileName, index_col=0)\n",
    "\n",
    "DRefRep2ResCols = {'Distance Min': 'Min Dist',\n",
    "                   'Distance Max': 'Max Dist',\n",
    "                   \n",
    "                   'Qual Equi': 'Qual Equi 1',\n",
    "                   'Qual Chi2': 'Qual Chi2+',\n",
    "                   'Qual DCV': 'Qual DCv+',\n",
    "                   'Qual KS': 'Qual KS+',\n",
    "                   \n",
    "                   'Grp Dist Tronc Gche': 'Groupe Tronc Gche',\n",
    "                   'Grp Dist Tronc Drte': 'Groupe Tronc Drte',\n",
    "                   \n",
    "                   'Meil AIC Tronc Id': 'Ordre Tronc Ident AIC',\n",
    "                   \n",
    "                   'Meil CKCv Tronc Proch'     : 'Ordre Tronc Proch Chi2 KS DCv',\n",
    "                   'Meil CVDens Tronc Proch'   : 'Ordre Tronc Proch DCv',\n",
    "                   'Meil Qual Equi Tronc Proch': 'Ordre Tronc Proch Qual Equi 1',\n",
    "                   'Meil Qual Chi2 Tronc Proch': 'Ordre Tronc Proch Qual Equi Chi2+',\n",
    "                   'Meil Qual KS Tronc Proch'  : 'Ordre Tronc Proch Qual Equi KS+',\n",
    "                   'Meil Qual DCV Tronc Proch' : 'Ordre Tronc Proch Qual Equi DCv+',\n",
    "                   \n",
    "                   'Ord CKCv'     : 'Ordre Global Chi2 KS DCv',\n",
    "                   'Ord Qual Equi': 'Ordre Global Qual Equi 1',\n",
    "                   'Ord Qual Chi2': 'Ordre Global Qual Equi Chi2+',\n",
    "                   'Ord Qual KS'  : 'Ordre Global Qual Equi KS+',\n",
    "                   'Ord Qual DCV' : 'Ordre Global Qual Equi DCv+',\n",
    "                   'Ord Simpl Tronc': 'Ordre Global DeltaAIC Chi2 KS DCv'}\n",
    "dfRefRep.rename(columns=DRefRep2ResCols, inplace=True)\n",
    "\n",
    "assert all(col in dfRefRep.columns for col in DRefRep2ResCols.values())\n",
    "\n",
    "dfRefRep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Compare loaded and post-computed to reference prototype intermediate report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results columns\n",
    "resFrCols = dfRes.columns\n",
    "', '.join(resFrCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns in results, but not in reference intermediate report\n",
    "diffCols = set(resFrCols) - set(dfRefRep.columns)\n",
    "assert diffCols == {'Ordre Global Qual Equi 2', 'Ordre Global Qual Equi 3',\n",
    "                    'Ordre Tronc Proch Qual Equi 2', 'Ordre Tronc Proch Qual Equi 3',\n",
    "                    'Qual Equi 2', 'Qual Equi 3'}\n",
    "diffCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "', '.join(sorted(resFrCols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "', '.join(sorted(dfRefRep.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns in reference intermediate report, but not in results\n",
    "diffCols = set(dfRefRep.columns) - set(resFrCols)\n",
    "assert not diffCols\n",
    "diffCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Index columns for comparison\n",
    "indexCols = [sampleNumCol] + sampleSelCols + [anlysIndCol, anlysAbbrevCol] + anlysParamCols\n",
    "', '.join(indexCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Round truncation distance parameters in order to be able to use them as part of the index columns\n",
    "# (Excel I/O changed some least significant after dot figures)\n",
    "dfRes['TrGche'] = dfRes['TrGche'].round(5)\n",
    "dfRes['TrDrte'] = dfRes['TrDrte'].round(5)\n",
    "\n",
    "dfRefRep['TrGche'] = dfRefRep['TrGche'].round(5)\n",
    "dfRefRep['TrDrte'] = dfRefRep['TrDrte'].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns to compare: let's ignore ...\n",
    "# * DeltaDCV et DeltaAIC: they depend on actual analyses sets done at once, may differing from ref to actual results,\n",
    "# * other string columns (comparison not implemented)\n",
    "# * other neglectible (run time, ... etc) or newly implemented (not in ref) columns\n",
    "subsetCols = [col for col in dfRefRep.columns \\\n",
    "              if col not in indexCols + ['HeureExec', 'DuréeExec', 'DossierExec',\n",
    "                                         'Fn Clé Mod', 'Sér Ajust Mod', 'Crit Chx Mod', 'Interv Conf',\n",
    "                                         'Fn Clé', 'Sér Ajust',\n",
    "                                         'Delta AIC', 'Delta CoefVar Densité',\n",
    "                                         'Max Dist', 'Min Dist',\n",
    "                                         'Qual Equi 2', 'Qual Equi 3',\n",
    "                                         'Ordre Tronc Proch Qual Equi 2', 'Ordre Tronc Proch Qual Equi 3',\n",
    "                                         'Ordre Global Qual Equi 2', 'Ordre Global Qual Equi 3']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Comparison 1: DataSet.compareDataFrames\n",
    "# => 21% analyses differing, and with only mostly-slightly different order indicators :-)\n",
    "#    (see histogram of order differences below)\n",
    "dfRelDiff = ads.DataSet.compareDataFrames(dfRes, dfRefRep, dropCloser=13, dropNans=True, dropCloserCols=True,\n",
    "                                          subsetCols=subsetCols, indexCols=indexCols)\n",
    "dict(refRows=len(dfRefRep), resRows=len(dfRes), diffRows=len(dfRelDiff), diffCols=len(dfRelDiff.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comparison 2: DataFrame.compare\n",
    "# Same diagnosis\n",
    "ordDiffCols = ['Ordre Tronc Proch DCv', 'Ordre Tronc Proch Qual Equi 1',\n",
    "               'Ordre Tronc Proch Qual Equi Chi2+', 'Ordre Tronc Proch Qual Equi KS+', 'Ordre Tronc Proch Qual Equi DCv+',\n",
    "               'Ordre Global Qual Equi Chi2+',\n",
    "               'Ordre Global Qual Equi KS+', 'Ordre Global Qual Equi DCv+', 'Ordre Global DeltaAIC Chi2 KS DCv']\n",
    "assert set(dfRelDiff.columns) == set(ordDiffCols)\n",
    "\n",
    "absCompCols = ['Analyse'] + ordDiffCols\n",
    "dfAbsDiff = dfRes[absCompCols].set_index('Analyse').sort_index() \\\n",
    "              .compare(dfRefRep[absCompCols].set_index('Analyse').sort_index())\n",
    "dfAbsDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check 1 order column (example)\n",
    "ordr = 'Ordre Tronc Proch DCv'\n",
    "dfAbsDiff.loc[(dfAbsDiff[(ordr, 'self')] - dfAbsDiff[(ordr, 'other')]).notnull(), [(ordr, 'self'), (ordr, 'other')]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute order column cell-by-cell differences\n",
    "dfAbsDeltaDiff = pd.DataFrame(index=dfAbsDiff.index)\n",
    "for ordr in ordDiffCols:\n",
    "    dfAbsDeltaDiff[ordr] = dfAbsDiff[(ordr, 'self')] - dfAbsDiff[(ordr, 'other')]\n",
    "dfAbsDeltaDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAbsDeltaDiff.max().max(), dfAbsDeltaDiff.min().min(), dfAbsDeltaDiff.notnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histogram for all order columns => mostly +/-1 differences\n",
    "hist, bins = np.histogram(dfAbsDeltaDiff.values, bins=int(dfAbsDeltaDiff.max().max() - dfAbsDeltaDiff.min().min()),\n",
    "                          range=(dfAbsDeltaDiff.min().min(), dfAbsDeltaDiff.max().max()))\n",
    "dfHist = pd.DataFrame(data=hist, index=bins[:-1])\n",
    "hist, bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict(nAnlys=len(dfRes), nDiffAnlys=len(dfAbsDiff), pctDiffAnlys=round(100*len(dfAbsDiff)/len(dfRes), 1),\n",
    "     pctDiffAnlys1=round(100 * dfHist.loc[-1:1].values.sum() / dfHist.loc[:, 0].sum(), 1),\n",
    "     pctDiffAnlys2=round(100 * dfHist.loc[-2:2].values.sum() / dfHist.loc[:, 0].sum(), 1),\n",
    "     pctDiffAnlys3=round(100 * dfHist.loc[-3:3].values.sum() / dfHist.loc[:, 0].sum(), 1),\n",
    "     pctDiffAnlys4=round(100 * dfHist.loc[-4:4].values.sum() / dfHist.loc[:, 0].sum(), 1),\n",
    "     pctDiffAnlys5=round(100 * dfHist.loc[-5:5].values.sum() / dfHist.loc[:, 0].sum(), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpn = dossier / f'{nomEtude}{sousEtude}-autofilsor-indicators-diffs.xlsx'\n",
    "with pd.ExcelWriter(fpn) as xlsWrtr:\n",
    "    dfRelDiff.to_excel(xlsWrtr, sheet_name='rel-diff')\n",
    "    dfAbsDiff.to_excel(xlsWrtr, sheet_name='abs-diff')\n",
    "    dfHist.to_excel(xlsWrtr, sheet_name='hist-diff')\n",
    "    \n",
    "fpn.as_posix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Quality and non-regression of filterSort*\n",
    "\n",
    "(prerequisite: run a.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deduplication algorithm params.\n",
    "R = results\n",
    "\n",
    "miDupSubsetDef = pd.MultiIndex.from_tuples([R.CLNObs, R.CLEffort, R.CLDeltaAic,\n",
    "                                            R.CLChi2, R.CLKS, R.CLCvMUw, R.CLCvMCw, R.CLDCv, \n",
    "                                            R.CLPDetec, R.CLPDetecMin, R.CLPDetecMax,\n",
    "                                            R.CLDensity, R.CLDensityMin, R.CLDensityMax])\n",
    "dDupRoundsDef = {R.CLDeltaAic: 1, R.CLChi2: 2, R.CLKS: 2, R.CLCvMUw: 2, R.CLCvMCw: 2, R.CLDCv: 2, \n",
    "                 R.CLPDetec: 3, R.CLPDetecMin: 3, R.CLPDetecMax: 3, R.CLDensity: 2, R.CLDensityMin: 2, R.CLDensityMax: 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Index columns for comparison\n",
    "anlysParamCols = ['Fn Clé Mod', 'Sér Ajust Mod', 'Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod']\n",
    "indexCols = [sampleNumCol] + sampleSelCols + [anlysIndCol] + anlysParamCols\n",
    "', '.join(indexCols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Load reference prototype final report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repFileName = workDir / f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-raptousech.ods'\n",
    "\n",
    "print(f'Fichier choisi : {repFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "ddfRefFinRep = pd.read_excel(repFileName, sheet_name=None)\n",
    "', '.join(ddfRefFinRep.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check that all sub-reports (1 per method) display the same columns\n",
    "assert all(ddfRefFinRep[meth].columns.to_list() == ddfRefFinRep['codexec'].columns.to_list()\n",
    "           for meth in ddfRefFinRep.keys() if meth.startswith('c') and meth != 'codexec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare ref. sub-reports for comparison :\n",
    "# * Drop pre-selection columns (added later in report module)\n",
    "# * Rename columns to \"industrialised\" names\n",
    "for meth in ddfRefFinRep:\n",
    "    if meth.startswith('c'):\n",
    "        ddfRefFinRep[meth].drop(columns=['Sélection finale', 'Sélection Qual Equi'], inplace=True)\n",
    "        ddfRefFinRep[meth].rename(columns=DRefRep2ResCols, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dBadAnalyses = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Check reference prototype final report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now some few checks ...\n",
    "#meth = 'ckcvqual975d8'\n",
    "#meth = 'ckcvqual950d10'\n",
    "meth = 'ckcvqual925d12'\n",
    "#meth = 'ckcvqual900d15'\n",
    "#meth = 'ckcvqual900d20'\n",
    "#meth = 'codexec'\n",
    "\n",
    "df = ddfRefFinRep[meth].copy()\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mainIndicCols = ['Delta AIC', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P', 'NObs', 'NTot Obs', 'CoefVar Densité', 'NbTot Pars']\n",
    "qualIndicCols = [col for col in df.columns if col.startswith('Qual')]\n",
    "resultCols = ['Densité', 'EDR/ESW', 'PDetec']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There should be no analysis with NaN values for main MCDS goodness params\n",
    "# (should be filtered out at first, just as ExecCode > 2 ones)\n",
    "df = df.loc[df[mainIndicCols].isnull().any(axis='columns'),\n",
    "            ['Analyse'] + mainIndicCols + qualIndicCols + resultCols]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# But there are, and seems they are all due to NaN Chi2 ...\n",
    "assert df['Chi2 P'].isnull().all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dBadAnalyses[meth] = df['Analyse'].to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Generate report (apply methods to test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mtoars = PrototypeConformResultsSet\n",
    "#mtoars = ads.MCDSTruncOptanalysisResultsSet\n",
    "\n",
    "whichBestQua = [mtoars.CLGrpOrdClTrChi2KSDCv, mtoars.CLGrpOrdClTrDCv, mtoars.CLGrpOrdClTrQuaBal1,\n",
    "                mtoars.CLGrpOrdClTrQuaChi2, mtoars.CLGrpOrdClTrQuaKS, mtoars.CLGrpOrdClTrQuaDCv]\n",
    "whichFinalQua = mtoars.CLCmbQuaBal1  # Was CLGrpOrdClTrQuaBal1, bad !!!\n",
    "\n",
    "filterSortReportSpecs = \\\n",
    "[dict(name='ExCode', \n",
    "      method=mtoars.filterSortOnExecCode,\n",
    "      deduplicate=dict(dupSubset=miDupSubsetDef, dDupRounds=dDupRoundsDef),\n",
    "      filterSort=dict(whichFinalQua=whichFinalQua, ascFinalQua=False)),\n",
    " dict(name='ExAicMQua-r{sightRate:.1f}d{nFinalRes}', \n",
    "      method=mtoars.filterSortOnExCAicMulQua,\n",
    "      deduplicate=dict(dupSubset=miDupSubsetDef, dDupRounds=dDupRoundsDef),\n",
    "      filterSort=dict(sightRate=92.5, nBestAIC=3, nBestQua=1, whichBestQua=whichBestQua,\n",
    "                      nFinalRes=12, whichFinalQua=whichFinalQua, ascFinalQua=False)),\n",
    " # ... etc.\n",
    " ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddfFinRep = dict()\n",
    "repLog = list()\n",
    "for spec in filterSortReportSpecs:\n",
    "\n",
    "    subRepName = spec['name'].format_map(spec['filterSort']).replace('.', '')\n",
    "    \n",
    "    iSubRep, subSteps = spec['method'](results, **spec['filterSort'], **spec['deduplicate'])\n",
    "    \n",
    "    ddfFinRep[subRepName] = results.dfTransData(lang='fr', index=iSubRep)\n",
    "    repLog += subSteps\n",
    "\n",
    "', '.join(ddfFinRep.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export to Excel.\n",
    "fpn = pl.Path('tmp') / f'{nomEtude}{sousEtude}-autofilsor-raptousech.xlsx'\n",
    "with pd.ExcelWriter(fpn) as xlsWrtr:\n",
    "    for subRepName, dfSubRepData in ddfFinRep.items():\n",
    "        dfSubRepData.to_excel(xlsWrtr, sheet_name=subRepName, index=True)            \n",
    "\n",
    "fpn.as_posix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. filterSortOn* checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Switch methods here ...\n",
    "#methIndus = 'ExecCode'\n",
    "methIndus = 'AicMQua-r925d12'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRes = ddfFinRep[methIndus].copy()\n",
    "dfRes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results columns\n",
    "resFrCols = dfRes.columns\n",
    "', '.join(resFrCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Round truncation distance parameters in order to be able to use them as part of the index columns\n",
    "# (Excel I/O changed some least significant after dot figures)\n",
    "dfRes['Dist Tronc Gche'] = dfRes['Dist Tronc Gche'].round(5)\n",
    "dfRes['Dist Tronc Drte'] = dfRes['Dist Tronc Drte'].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There should be no analysis with NaN values for main MCDS goodness params\n",
    "# (should be filtered out at first, just as ExecCode > 2 ones)\n",
    "_mainIndicCols = [col for col in mainIndicCols if col in dfRes.columns]\n",
    "_qualIndicCols = [col for col in qualIndicCols if col in dfRes.columns]\n",
    "_resultCols = [col for col in resultCols if col in df.columns]\n",
    "\n",
    "df = dfRes.loc[dfRes[_mainIndicCols].isnull().any(axis='columns'), ['Analyse'] + _mainIndicCols + _qualIndicCols + _resultCols]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# But there are, and seems they are all due to NaN Chi2 ...\n",
    "assert df['Chi2 P'].isnull().all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Are there all the same ones as in Ref report ?\n",
    "# * ExecCode: no (46 not in common)\n",
    "# * AicCKCvQua-r925d12: yes !\n",
    "len(set(df['Analyse']) - set(dBadAnalyses[meth])), len(set(dBadAnalyses[meth]) - set(df['Analyse']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get reference sub-report and check that all its columns are also in the \"industrialised\" sub-report\n",
    "dfRefRep = ddfRefFinRep[meth]\n",
    "assert all(col in resFrCols for col in dfRefRep.columns)\n",
    "\n",
    "', '.join(dfRefRep.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Round truncation distance parameters in order to be able to use them as part of the index columns\n",
    "# (Excel I/O changed some least significant after dot figures)\n",
    "dfRefRep['Dist Tronc Gche'] = dfRefRep['Dist Tronc Gche'].round(5)\n",
    "dfRefRep['Dist Tronc Drte'] = dfRefRep['Dist Tronc Drte'].round(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns to compare: let's ignore ...\n",
    "# * DeltaAIC: depends on actual analyses sets done at once, may be differing from ref to actual results,\n",
    "# * other duplicate columns (analysis params)\n",
    "subsetCols = [col for col in dfRefRep.columns\n",
    "              if col not in indexCols + ['Delta AIC', 'FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Comparaison => bilan = 28% d'analyses non présentes dans les 2 rapports ou à valeurs différentes\n",
    "# C'est assez cohérent avec les 21% présentant des différences de classement par ordre d'indicateurs ...\n",
    "dfRelDiff = ads.DataSet.compareDataFrames(dfRefRep, dfRes, indexCols=indexCols, subsetCols=subsetCols,\n",
    "                                          dropCloser=6, dropNans=True, dropCloserCols=True)\n",
    "#assert len(dfRelDiff) == 0\n",
    "dict(refRows=len(dfRefRep), resRows=len(dfRes), diffRows=len(dfRelDiff), diffCols=len(dfRelDiff.columns),\n",
    "     pctDiff=round(100 * len(dfRelDiff) / len(dfRes), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff.to_excel('tmp/{}-reldiff.xlsx'.format(meth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List diffing analyses that are not in the 2 reports\n",
    "def allNullOrZero(s):\n",
    "    return sum(s.isnull() | (s == 0)) == len(s)\n",
    "\n",
    "df = dfRelDiff.loc[dfRelDiff.apply(allNullOrZero, axis='columns')]\n",
    "print('pctNotBoth:', round(100 * len(df) / len(dfRes), 1))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove them => remains those in the 2 reports, but with diffs\n",
    "df = dfRelDiff.drop(dfRelDiff.loc[dfRelDiff.apply(allNullOrZero, axis='columns')].index)\n",
    "print('pctBothButDiffs:', round(100 * len(df) / len(dfRes), 1))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove bad analyses (with NaN in main MCDS results)\n",
    "df = df.dropna(axis='index', how='any', subset=[col for col in _mainIndicCols if col in df.columns])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove columns with rel diffs lower than 1e-6 all along\n",
    "df = df.drop(columns=[col for col in df.columns if df[col].gt(6).all()])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# And there only remains 'Ordre *' columns\n",
    "assert all(col.startswith('Ordre') for col in df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# At the end good news : differences are only due to :\n",
    "# * see above non-regression tests on enriched results/reports :\n",
    "#    (only) partially explained order differences (but mostly small) => different lists of analyses (20-25%)\n",
    "# * prototype bug that keeps in the race ... bad analyses with NaN as main MCDS goodness indicator values (Chi2 ... etc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel('tmp/{}-reldiff-both-but-nan-or-diff6.xlsx'.format(meth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Quality tests of (industrialised) filterSort*\n",
    "\n",
    "(prerequisite: run 1. above)\n",
    "\n",
    "1. load results with the real industrialised MCDSTruncationOptanalysisResultsSet\n",
    "2. apply filter-sort methods\n",
    "3. check output quality\n",
    "    * TODO: define what to check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddfSubRep['ExecCode']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddfSubRep['AicCKCvQua-r925d12']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repLog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Development : Optimise MCDSAnalyser._postComputeQualityIndicators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Prepare stuff for creating MCDSAnalysisResultsSet objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source / Results data\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'Distance'\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'NumEchant'\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "\n",
    "sampleAbbrevCol = 'AbrevEchant'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')\n",
    "\n",
    "# General DS analysis parameters\n",
    "varIndCol = 'NumAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Hectare'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "clustering = False\n",
    "\n",
    "# Results post-computation parameters\n",
    "ldTruncIntrvSpecs = [dict(col='left', minDist=5.0, maxLen=5.0), dict(col='right', minDist=25.0, maxLen=25.0)]\n",
    "truncIntrvEpsilon = 1e-6\n",
    "\n",
    "# Load individualised observations and actual transects\n",
    "indivObsFile = 'refin/ACDC2019-Naturalist-ExtraitObsIndiv.ods'\n",
    "\n",
    "dfObsIndiv = ads.DataSet(indivObsFile, sheet='DonnéesIndiv').dfData\n",
    "\n",
    "dfTransects = ads.DataSet(indivObsFile, sheet='Inventaires').dfData\n",
    "\n",
    "dict(indivObs=len(dfObsIndiv), transects=len(dfTransects))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What's better to create an MCDSAnalysisResultsSet objets than a MCDSAnalyser instance ?\n",
    "anlr = \\\n",
    "    ads.MCDSAnalyser(dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                     transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                     sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                     abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                     anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                     distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                     surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                     resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                          after=anlysParamCols + [anlysAbbrevCol]),\n",
    "                     ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. _postComputeQualityIndicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "# Note: Okay, it's actually an MCDSTruncOptAnalysisResultsSet file ... but we'll ignore the extra columns, promised :-)\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = anlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResultsOpt1(ads.MCDSAnalysisResultsSet):\n",
    "    \n",
    "    Super = ads.MCDSAnalysisResultsSet\n",
    "    \n",
    "    def __init__(self, results):\n",
    "        \n",
    "        self._dfData = results._dfData\n",
    "    \n",
    "    # Post computations : Quality indicators.\n",
    "    DNormKeyFn = dict(HNORMAL=1.0, UNIFORM=0.9, HAZARD=0.6, NEXPON=0.1)\n",
    "    # DNormKeyFn = dict(HNORMAL=1.00, UNIFORM=0.75, HAZARD=0.5, NEXPON=0.1)  # Not better\n",
    "\n",
    "    CLsQuaIndicSources = [Super.CLKeyFn, Super.CLNAdjPars, Super.CLNTotPars, Super.CLNObs, Super.CLNTotObs,\n",
    "                          Super.CLChi2, Super.CLKS, Super.CLCvMUw, Super.CLCvMCw, Super.CLDCv]\n",
    "    \n",
    "    CIKeyFn = CLsQuaIndicSources.index(Super.CLKeyFn)\n",
    "    CINAdjPars = CLsQuaIndicSources.index(Super.CLNAdjPars)\n",
    "    CINTotPars = CLsQuaIndicSources.index(Super.CLNTotPars)\n",
    "    CINObs = CLsQuaIndicSources.index(Super.CLNObs)\n",
    "    CINTotObs = CLsQuaIndicSources.index(Super.CLNTotObs)\n",
    "    CIChi2 = CLsQuaIndicSources.index(Super.CLChi2)\n",
    "    CIKS = CLsQuaIndicSources.index(Super.CLKS)\n",
    "    CICvMUw = CLsQuaIndicSources.index(Super.CLCvMUw)\n",
    "    CICvMCw = CLsQuaIndicSources.index(Super.CLCvMCw)\n",
    "    CIDCv = CLsQuaIndicSources.index(Super.CLDCv)\n",
    "    \n",
    "    @classmethod\n",
    "    def _combinedQualityBalanced1(cls, aRes):  # The one used for ACDC 2019 filtering & sorting in jan/feb 2021\n",
    "\n",
    "        # MCDS results (or so).\n",
    "        chi2 = aRes[cls.CIChi2]\n",
    "        ks = aRes[cls.CIKS]\n",
    "        chi2 = aRes[cls.CIChi2]\n",
    "        chi2KsCvMs = aRes[cls.CIChi2:cls.CICvMCw + 1].prod()\n",
    "        normNObs = aRes[cls.CINObs] / aRes[cls.CINTotObs]\n",
    "        normNTotPars = 1 / (0.2 * max(2, aRes[cls.CINTotPars]) + 0.6)\n",
    "        normCVDens = math.exp(-12 * aRes[cls.CIDCv] * aRes[cls.CIDCv])\n",
    "        \n",
    "        return (chi2KsCvMs * normNObs * normNTotPars * normCVDens) ** (1.0/7)\n",
    "\n",
    "    @classmethod\n",
    "    def _combinedQualityAll(cls, aRes):\n",
    "        \n",
    "        \"\"\"Does NOT work, because of a pandas 1.1+ regression on DataFrame.apply(..., raw=True, ...),\n",
    "        apparently not fixed yet (see https://github.com/pandas-dev/pandas/issues/34822) :\n",
    "        \n",
    "        ValueError                                Traceback (most recent call last)\n",
    "        C:\\PortableApps\\MiniConda3\\envs\\py38\\lib\\site-packages\\pandas\\core\\internals\\managers.py in create_block_manager_from_blocks(blocks, axes)\n",
    "           1674                 blocks = [\n",
    "        -> 1675                     make_block(\n",
    "           1676                         values=blocks[0], placement=slice(0, len(axes[0])), ndim=2\n",
    "\n",
    "        C:\\PortableApps\\MiniConda3\\envs\\py38\\lib\\site-packages\\pandas\\core\\internals\\blocks.py in make_block(values, placement, klass, ndim, dtype)\n",
    "           2750 \n",
    "        -> 2751     return klass(values, ndim=ndim, placement=placement)\n",
    "           2752 \n",
    "\n",
    "        C:\\PortableApps\\MiniConda3\\envs\\py38\\lib\\site-packages\\pandas\\core\\internals\\blocks.py in __init__(self, values, placement, ndim)\n",
    "            141         if self._validate_ndim and self.ndim and len(self.mgr_locs) != len(self.values):\n",
    "        --> 142             raise ValueError(\n",
    "            143                 f\"Wrong number of items passed {len(self.values)}, \"\n",
    "\n",
    "        ValueError: Wrong number of items passed 5, placement implies 10\n",
    "        \n",
    "        See also https://stackoverflow.com/questions/67678210/raw-true-causes-valueerror-in-pandas-dataframe-apply\n",
    "        for description and code for reproducing.\n",
    "        \"\"\"\n",
    "        \n",
    "        # MCDS results (or so).\n",
    "        chi2 = aRes[cls.CIChi2]\n",
    "        ks = aRes[cls.CIKS]\n",
    "        chi2 = aRes[cls.CIChi2]\n",
    "        chi2KsCvMs = aRes[cls.CIChi2:cls.CICvMCw + 1].prod()\n",
    "        normNObs = aRes[cls.CINObs] / aRes[cls.CINTotObs]\n",
    "        \n",
    "        # October 2021\n",
    "        normKeyFn = cls.DNormKeyFn.get(aRes[cls.CIKeyFn], 0.0)\n",
    "\n",
    "        # A more devaluating version for NAdjPars, CVDens, also using KeyFn\n",
    "        normNAdjPars = math.exp(-0.15 * aRes[cls.CINAdjPars] * aRes[cls.CINAdjPars])\n",
    "        normCVDens = math.exp(-20 * aRes[cls.CIDCv] * aRes[cls.CIDCv])\n",
    "        prodAll8NormSrcIndics = chi2KsCvMs * normNObs * normNAdjPars * normCVDens * normKeyFn\n",
    "        quaBal2 = prodAll8NormSrcIndics ** 0.125\n",
    "\n",
    "        # An even more devaluating version for NAdjPars, CVDens, also using \n",
    "        normNAdjPars = math.exp(-0.17 * aRes[cls.CINAdjPars] * aRes[cls.CINAdjPars])\n",
    "        normCVDens = math.exp(-63 * aRes[cls.CIDCv] ** 2.8)\n",
    "        prodAll8NormSrcIndics = chi2KsCvMs * normNObs * normNAdjPars * normCVDens * normKeyFn\n",
    "        quaBal3 = prodAll8NormSrcIndics ** 0.125\n",
    "\n",
    "        # Follow _combinedQualityBalanced3 update (were based on _combinedQualityBalanced1)\n",
    "        moreChi2 = (prodAll8NormSrcIndics * chi2) ** (1.0/9)\n",
    "        moreKS = (prodAll8NormSrcIndics * ks) ** (1.0/9)\n",
    "        moreDCv = (prodAll8NormSrcIndics * normCVDens) ** (1.0/9)\n",
    "        \n",
    "        return quaBal2, quaBal3, moreChi2, moreKS, moreDCv  # Must be same order as in CLsNewQuaIndics !\n",
    "\n",
    "    CLsNewQuaIndics = [Super.CLCmbQuaBal2, Super.CLCmbQuaBal3, Super.CLCmbQuaChi2, Super.CLCmbQuaKS, Super.CLCmbQuaDCv]\n",
    "\n",
    "    def _postComputeQualityIndicators(self):\n",
    "        \n",
    "        \"\"\"Does not work because of a pandas 1.1+ bug, see above _combinedQualityAllOpt1\"\"\"\n",
    "        \n",
    "        cls = self\n",
    "\n",
    "        logger.debug('Post-computing Quality Indicators (opt1)')\n",
    "\n",
    "        self._dfData[cls.CLSightRate] = 100 * self._dfData[cls.CLNObs] / self._dfData[cls.CLNTotObs]  # [0,1] => %\n",
    "\n",
    "        # Prepare data for computations\n",
    "        logger.debug1('* Pre-processing source data')\n",
    "\n",
    "        # a. extract the useful columns, after adding them if not present\n",
    "        #    (NaN value, except for CLKeyFn, that MUST be there anyway)\n",
    "        for miCol in cls.CLsQuaIndicSources:\n",
    "            if miCol not in self._dfData.columns and miCol != cls.CLKeyFn:\n",
    "                self._dfData[miCol] = np.nan\n",
    "        dfCompData = self._dfData[cls.CLsQuaIndicSources].copy()\n",
    "\n",
    "        # b. historical bal qua 1\n",
    "        logger.debug1('* Balanced quality 1')\n",
    "        self._dfData[cls.CLCmbQuaBal1] = dfCompData.apply(cls._combinedQualityBalanced1, axis='columns')\n",
    "\n",
    "        # c. newer quality indicators\n",
    "        #    (NaN value MUST kill down these indicators to compute => we have to enforce this)\n",
    "        dfCompData.fillna({cls.CLNObs: cls.KilrNObs,\n",
    "                           cls.CLChi2: cls.KilrStaTest, cls.CLKS: cls.KilrStaTest,\n",
    "                           cls.CLCvMUw: cls.KilrStaTest, cls.CLCvMCw: cls.KilrStaTest,\n",
    "                           cls.CLDCv: cls.KilrDensCv,  # Usually considered good under 0.3\n",
    "                           cls.CLNTotObs: cls.KilrNTotObs,  # Should slap down _normObs whatever NObs\n",
    "                           cls.CLNAdjPars: cls.KilrNPars,  # Should slap down _normNAdjPars whatever NObs\n",
    "                           cls.CLNTotPars: cls.KilrNPars},\n",
    "                          inplace=True)\n",
    "\n",
    "        logger.debug1('* Balanced quality 2, 3, Chi2+, KS+, DCv+')\n",
    "        self._dfData.drop(columns=cls.CLsNewQuaIndics, inplace=True, errors='ignore')  # Cleanup\n",
    "        aNewQuaIndics = dfCompData.apply(cls._combinedQualityAll, axis='columns', raw=True, result_type='reduce').values\n",
    "        self._dfData = self._dfData.join(pd.DataFrame(aNewQuaIndics, index=self._dfData.index,\n",
    "                                                      columns=pd.MultiIndex.from_tuples(cls.CLsNewQuaIndics)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResultsOpt2(ResultsOpt1):\n",
    "    \n",
    "    Super = ResultsOpt1\n",
    "    \n",
    "    def __init__(self, results):\n",
    "        \n",
    "        super().__init__(results)\n",
    "    \n",
    "    @classmethod\n",
    "    def _combinedQualityBalanced1(cls, aRes):  # The one used for ACDC 2019 filtering & sorting in jan/feb 2021\n",
    "\n",
    "        chi2KsCvMs = aRes[:, cls.CIChi2:cls.CICvMCw + 1].prod(axis=1)\n",
    "        normNObs = aRes[:, cls.CINObs].astype(float) / aRes[:, cls.CINTotObs].astype(float)\n",
    "        normNTotPars = 1 / (0.2 * np.maximum(2, aRes[:, cls.CINTotPars].astype(float)) + 0.6)\n",
    "        normCVDens = np.exp(-12 * np.square(aRes[:, cls.CIDCv].astype(float)))\n",
    "\n",
    "        return np.power(chi2KsCvMs * normNObs * normNTotPars * normCVDens, 1 / 7.0) # shape: aRes rows, 1 column\n",
    "\n",
    "    ufnNormKeyFn = np.frompyfunc(lambda keyFn: ResultsOpt2.DNormKeyFn.get(keyFn, 0.0), 1, 1)\n",
    "    \n",
    "    @classmethod\n",
    "    def _combinedQualityAll(cls, aRes):\n",
    "        \n",
    "        chi2 = aRes[:, cls.CIChi2].astype(float)\n",
    "        ks = aRes[:, cls.CIKS].astype(float)\n",
    "        dcv = aRes[:, cls.CIDCv].astype(float)\n",
    "        chi2KsCvMs = aRes[:, cls.CIChi2:cls.CICvMCw + 1].astype(float).prod(axis=1)\n",
    "        normNObs = aRes[:, cls.CINObs].astype(float) / aRes[:, cls.CINTotObs].astype(float)\n",
    "\n",
    "        # October 2021\n",
    "        nAdjPars2 = np.square(aRes[:, cls.CINAdjPars]).astype(float)\n",
    "        normKeyFn = cls.ufnNormKeyFn(aRes[:, cls.CIKeyFn])\n",
    "        normChi2KsCvMsNObsKFn = chi2KsCvMs * normNObs * normKeyFn\n",
    "\n",
    "        # QualBal2 : A more devaluating version for NAdjPars, CVDens, also using KeyFn\n",
    "        normNAdjPars2 = np.exp(-0.15 * nAdjPars2)\n",
    "        normCVDens2 = np.exp(-20 * np.square(dcv))\n",
    "        quaBal2 = np.power(normChi2KsCvMsNObsKFn * normNAdjPars2 * normCVDens2, 1 / 8.0)\n",
    "\n",
    "        # QualBal3 : An even more devaluating version for NAdjPars, CVDens, also using \n",
    "        normNAdjPars3 = np.exp(-0.17 * nAdjPars2)\n",
    "        normCVDens3 = np.exp(-63 * np.power(dcv, 2.8))\n",
    "        normChi2KsCvMsNObsKFnAdjPDcv3 = normChi2KsCvMsNObsKFn * normNAdjPars3 * normCVDens3\n",
    "        quaBal3 = np.power(normChi2KsCvMsNObsKFnAdjPDcv3, 1 / 8.0)\n",
    "\n",
    "        # QualMoreX : Follow _combinedQualityBalanced3 update (were based on _combinedQualityBalanced1)\n",
    "        moreChi2 = np.power(normChi2KsCvMsNObsKFnAdjPDcv3 * chi2, 1 / 9.0)\n",
    "        moreKS = np.power(normChi2KsCvMsNObsKFnAdjPDcv3 * ks, 1 / 9.0)\n",
    "        moreDCv = np.power(normChi2KsCvMsNObsKFnAdjPDcv3 * normCVDens3, 1 / 9.0)\n",
    "        \n",
    "        return quaBal2, quaBal3, moreChi2, moreKS, moreDCv  # shape: aRes rows, 1 column each + order of CLsNewQuaIndics !\n",
    "        \n",
    "    def _postComputeQualityIndicators(self):\n",
    "               \n",
    "        cls = self\n",
    "\n",
    "        logger.debug('Post-computing Quality Indicators (opt2)')\n",
    "\n",
    "        self._dfData[cls.CLSightRate] = 100 * self._dfData[cls.CLNObs] / self._dfData[cls.CLNTotObs]  # [0,1] => %\n",
    "\n",
    "        # Prepare data for computations\n",
    "        logger.debug1('* Pre-processing source data')\n",
    "\n",
    "        # a. extract the useful columns, after adding them if not present\n",
    "        #    (NaN value, except for CLKeyFn, that MUST be there anyway)\n",
    "        for miCol in cls.CLsQuaIndicSources:\n",
    "            if miCol not in self._dfData.columns and miCol != cls.CLKeyFn:\n",
    "                self._dfData[miCol] = np.nan\n",
    "        dfCompData = self._dfData[cls.CLsQuaIndicSources].copy()\n",
    "\n",
    "        # b. historical bal qua 1\n",
    "        logger.debug1('* Balanced quality 1')\n",
    "        self._dfData[cls.CLCmbQuaBal1] = cls._combinedQualityBalanced1(dfCompData.values)\n",
    "\n",
    "        # c. newer quality indicators\n",
    "        #    (NaN value MUST kill down these indicators to compute => we have to enforce this)\n",
    "        dfCompData.fillna({cls.CLNObs: cls.KilrNObs,\n",
    "                           cls.CLChi2: cls.KilrStaTest, cls.CLKS: cls.KilrStaTest,\n",
    "                           cls.CLCvMUw: cls.KilrStaTest, cls.CLCvMCw: cls.KilrStaTest,\n",
    "                           cls.CLDCv: cls.KilrDensCv,  # Usually considered good under 0.3\n",
    "                           cls.CLNTotObs: cls.KilrNTotObs,  # Should slap down _normObs whatever NObs\n",
    "                           cls.CLNAdjPars: cls.KilrNPars,  # Should slap down _normNAdjPars whatever NObs\n",
    "                           cls.CLNTotPars: cls.KilrNPars},\n",
    "                          inplace=True)\n",
    "\n",
    "        logger.debug1('* Balanced quality 2, 3, Chi2+, KS+, DCv+')\n",
    "        self._dfData[cls.CLsNewQuaIndics] = np.stack(cls._combinedQualityAll(dfCompData.values), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "# Initial version => 975 +/- 11 ms on a 6-HT-core i7-10850H\n",
    "results._postComputeQualityIndicators()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #1 optimised version (does not work)\n",
    "#opt1Res = ResultsOpt1(results.copy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "#opt1Res._postComputeQualityIndicators()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #2 optimised version\n",
    "opt2Res = ResultsOpt2(results.copy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "\n",
    "opt2Res._postComputeQualityIndicators()\n",
    "\n",
    "# => 9.5 +/- 0.2 ms on a 6-HT-core i7-10850H => a x100 boost :-)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare to original method results\n",
    "quaIndCols = [results.CLCmbQuaBal1] + opt2Res.CLsNewQuaIndics\n",
    "\n",
    "results._dfData[quaIndCols].compare(opt2Res._dfData[quaIndCols])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ads.DataSet.compareDataFrames(results._dfData, opt2Res._dfData,\n",
    "                                     indexCols=[('header (head)', 'NumAnlys', 'Value')],\n",
    "                                     subsetCols=quaIndCols, dropCloser=15, dropCloserCols=True).empty\n",
    "\n",
    "# Success 2021-11-28 PM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results._dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Development : Automated filtering and sorting of optanalysis results\n",
    "\n",
    "En entrée :\n",
    "* soit : export Excel des résultats d'opt-analyses (via [Visionature-ds-point / XVI. Analyses automatiques / 2a ou 2b](../Visionature-ds-point.ipynb#XVI.-Analyses-automatiques)),\n",
    "* soit : rapport Excel 'full' des résultats d'opt-analyses généré une autre fois (via b. ci-dessous).\n",
    "\n",
    "N.B. Code historique de développement et d'essais, maintenant industrialisé et enrichi via MCDSTruncationOptanalysisResultsSet et MDCDResultsFilterSortReport"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Chargement / Génération des données d'entrée"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Choix étude / sous-étude / variante\n",
    "\n",
    "(Cf. [Visionature-ds-point / I. Paramètres de l'étude : import / filtrage des données](../Visionature-ds-point.ipynb#I.-Param%C3%A8tres-de-l'%C3%A9tude-%3A-import-%2F-filtrage-des-donn%C3%A9es%2C-...) et [Visionature-ds-point / XVI. 1. c. Optanalyses à faire : variante d'études](../Visionature-ds-point.ipynb#c.-Analyses-%C3%A0-faire-%3A-variante-d'%C3%A9tudes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nomEtude = 'ACDC2019'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sousEtude = '-Nat'\n",
    "#sousEtude = '-Pap'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "varEtude = ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Si pas déjà disponible, génération du rapport Excel 'Full' des optanalyses à partir d'un export Excel des résultats\n",
    "\n",
    "(à partir des résultats exportés en Excel via [Visionature-ds-point / XVI. Analyses automatiques / 2a ou 2b](../Visionature-ds-point.ipynb#XVI.-Analyses-automatiques))\n",
    "\n",
    "N.B. Si ces résultats ont été générés via pyaudisam >= 12/08/2021, la suite n'a pas grand intérêt, puisqu'elle produit les informations et la mise en forme qui sont maintenant auto-calculées / faites par MCDSTruncationOptanalysisResultsSet et MDCDResultsFilterSortReport."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Liste des dossiers/résultats disponibles pour l'étude / sous-étude / variante.\n",
    "resFolders = [fn.name for fn in dossier.glob('[0-9]'*6+'-'+'[0-9]'*4)\n",
    "              if (fn / f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-resultats.xlsx').is_file()]\n",
    "\n",
    "print('Résultats disponibles:', ', '.join(resFolders))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choix du résultats à traiter\n",
    "workDir = dossier / resFolders[0]  # <=== Choisir le dossier de résultats ici.\n",
    "\n",
    "resFileName = workDir / f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-resultats.xlsx'\n",
    "\n",
    "print(f'Fichier choisi : {resFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chargement des résultats\n",
    "optanlr = \\\n",
    "    ads.MCDSTruncationOptanalyser(dfObsCatIndiv, dfTransects=dfTransects,\n",
    "                                  effortConstVal=effortConst, dSurveyArea=dZoneEtude, \n",
    "                                  transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                                  sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                                  abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                                  anlysIndCol=anlysIndCol, sampleIndCol=sampleNumCol,\n",
    "                                  distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                                  surveyType=surveyType, distanceType=distanceType, clustering=groupage,\n",
    "                                  resultsHeadCols=dict(before=[anlysIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                                       #after=anlysParamCols + [optimTruncCol, anlysAbbrevCol]))\n",
    "                                                       after=anlysParamCols + [anlysAbbrevCol])) # TODO: test !\n",
    "\n",
    "results = optanlr.setupResults()\n",
    "\n",
    "results.fromExcel(resFileName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sélection des colonnes pour les tableaux de synthèse du rapport\n",
    "synthRepCols = \\\n",
    "[('header (head)', col, 'Value') for col in [anlysIndCol, sampleNumCol]] \\\n",
    "+ [('header (sample)', col, 'Value') for col in samplingCols] \\\n",
    "+ [('parameters', 'estimator key function', 'Value'),\n",
    "   ('parameters', 'estimator adjustment series', 'Value'),\n",
    "   ('parameters', 'left truncation distance', 'Value'),\n",
    "   ('parameters', 'right truncation distance', 'Value'),\n",
    "   ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "   \n",
    "   ('run output', 'run status', 'Value'),\n",
    "   \n",
    "   ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "   ('encounter rate', 'right truncation distance (w)', 'Value'),\n",
    "   ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "   \n",
    "   ('detection probability', 'Delta AIC', 'Value'),\n",
    "   ('detection probability', 'AIC value', 'Value'),\n",
    "   ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "   ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "   ('detection probability', 'Cramér-von Mises (uniform weighting) test probability', 'Value'),\n",
    "   ('detection probability', 'Cramér-von Mises (cosine weighting) test probability', 'Value'),\n",
    "   ('density/abundance', 'density of animals', 'Cv'),\n",
    "   \n",
    "   ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "   ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Lcl'),\n",
    "   ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Ucl'),\n",
    "   \n",
    "   ('density/abundance', 'density of animals', 'Value'),\n",
    "   ('density/abundance', 'density of animals', 'Lcl'),\n",
    "   ('density/abundance', 'density of animals', 'Ucl'),\n",
    "   ('density/abundance', 'density of animals', 'Delta Cv'),\n",
    "   \n",
    "   ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "   ('detection probability', 'probability of detection (Pw)', 'Lcl'),\n",
    "   ('detection probability', 'probability of detection (Pw)', 'Ucl'),\n",
    "   ('detection probability', 'probability of detection (Pw)', 'Df'),\n",
    "\n",
    "   ('density/abundance', 'number of animals, if survey area is specified', 'Value'),\n",
    "   ('density/abundance', 'number of animals, if survey area is specified', 'Lcl'),\n",
    "   ('density/abundance', 'number of animals, if survey area is specified', 'Ucl'),\n",
    "   ('density/abundance', 'number of animals, if survey area is specified', 'Df'),\n",
    "   \n",
    "   ('run output', 'run folder', 'Value')\n",
    "]\n",
    "\n",
    "sortRepCols = \\\n",
    "[('header (head)', sampleNumCol, 'Value')] \\\n",
    "+ [('header (sample)', col, 'Value') for col in samplingCols] \\\n",
    "+ [('parameters', 'left truncation distance', 'Value'),\n",
    "   ('parameters', 'right truncation distance', 'Value'),\n",
    "   ('detection probability', 'Delta AIC', 'Value'),\n",
    "   ('detection probability', 'chi-square test probability determined', 'Value'), # For same AIC !\n",
    "   ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'), # For same Chi2 !\n",
    "   ('run output', 'run status', 'Value'), # For same KS !\n",
    "]\n",
    "#   ('density/abundance', 'density of animals', 'Delta Cv')]\n",
    "\n",
    "sortRepAscend = [True]*(1+len(samplingCols)+3) + [False]*2 + [True]\n",
    "\n",
    "assert len(sortRepCols) == len(sortRepAscend)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = ads.MCDSResultsFullReport(resultsSet=results, synthCols=synthRepCols,\n",
    "                                   sortCols=sortRepCols, sortAscend=sortRepAscend,\n",
    "                                   title=titreEtude, subTitle='Rapport d\\'analyse',\n",
    "                                   anlysSubTitle='Détail des analyses', description=descrEtude,\n",
    "                                   keywords=motsClesEtude, pySources=['Visionature-ds-points.ipynb'],\n",
    "                                   lang='fr', plotImgSize=(768, 384),\n",
    "                                   #plotImgQuality=80, plotImgFormat='jpg', # Same final size as raw PNG :-(\n",
    "                                   tgtFolder=workDir, tgtPrefix=f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-rapport')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "xlsxRep = report.toExcel()\n",
    "\n",
    "HTML(f'Rapport Excel : <a href=\"{xlsxRep}\" target=\"blank\">{xlsxRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#\n",
    "#htmlRep = report.toHtml()\n",
    "#\n",
    "#HTML(f'Rapport HTML : <a href=\"{htmlRep}\" target=\"blank\">{htmlRep}</a>')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Sélection et chargement du rapport 'Full' Excel à traiter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[XVI.1.c. Analyses à faire : variante d'études](#c.-Analyses-%C3%A0-faire-%3A-variante-d'%C3%A9tudes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn = f'{nomEtude}{sousEtude}-OptAnalyses{varEtude}-rapport.xlsx'\n",
    "print(fn)\n",
    "repFolders = [dn.name for dn in dossier.glob('[0-9]'*6+'-'+'[0-9]'*4)\n",
    "              if (dn / fn).is_file()]\n",
    "\n",
    "print('Rapports disponibles:', ', '.join(repFolders))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "repFolder = dossier / repFolders[0]  # <=== Choisir le dossier du rapport à exploiter ici.\n",
    "\n",
    "xlsxRep = repFolder / fn\n",
    "    \n",
    "print(f'Fichier choisi : {xlsxRep.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRep = pd.read_excel(xlsxRep, sheet_name='Détails', index_col=0)\n",
    "dfRep.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRep.columns, len(dfRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfExplOptAnlysSpecs = ads.DSAnalyser.explicitVariantSpecs(optAnlysSpecs) \n",
    "#dfExplOptAnlysSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Enrichissement\n",
    "\n",
    "* données absentes des réusltats : nb total d'individus contactés par échantillon, distance max par ééchantillon,\n",
    "* indicateurs qualité supplémentaires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfSampleStats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Données absentes du rapport en entrée\n",
    "\n",
    "(TODO: à ajouter automatiquement)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stats sur les échantillons (nbre d'individus, distances max)\n",
    "dfStatsEch = dfRep[['Echant'] + indexCols].drop_duplicates().set_index('Echant')\n",
    "dfStatsEch = dfStatsEch.join(dfSampleStats.set_index(indexCols), on=indexCols)\n",
    "dfStatsEch.insert(0, 'Abréviation', dfStatsEch[indexCols].apply(sampleAbbrev, axis='columns'))\n",
    "dfStatsEch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfStatsEch.reset_index().to_excel('tmp/stats-ech.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRep = dfRep.join(dfStatsEch[indexCols + ['Distance Min', 'Distance Max', 'NTot Obs']].set_index(indexCols), on=indexCols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Taux d'individus conservés par les troncatures\n",
    "dfRep['Taux Obs'] = dfRep.apply(lambda s: 100 * s['NObs'] / s['NTot Obs'], axis='columns')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Obsolète : détection des résultats avec troncatures optimisées\n",
    "\n",
    "Maintenant (27/12/2020), une colonne pour ça existe en sortie de l'optanalyseur."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimTruncCol = ads.MCDSTruncationOptanalyser.OptimTruncFlagCol\n",
    "\n",
    "if optimTruncCol not in dfRep.columns or dfRep[optimTruncCol].isnull().all():\n",
    "    \n",
    "    def isTruncationOptimised(sRes):  # np.modf(x, 1) => decimal part of x\n",
    "        return 1 if sRes[['Dist Tronc Gche', 'Dist Tronc Drte']].fillna(0).mod(1).sum() > 0 \\\n",
    "                    or (sRes[['Dist Tronc Gche', 'Dist Tronc Drte']].isnull().all() \n",
    "                        and not pd.isnull(sRes['Tranch Dist Mod'])) \\\n",
    "                 else 0\n",
    "    dfRep[optimTruncCol] = dfRep.apply(isTruncationOptimised, axis='columns')\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print(f'Colonne \"{optimTruncCol}\" déja présente dans le rapport, rien à faire de plus')\n",
    "\n",
    "len(dfRep), dfRep[optimTruncCol].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Des indicateurs qualité composés\n",
    "\n",
    "(différentes recette privilégiant ou pas tel ou tel indicateur de base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normNObs(sRes):\n",
    "    return sRes['NObs'] / sRes['NTot Obs']\n",
    "\n",
    "def normNTotPars(sRes, a=0.2, b=0.6):\n",
    "    #return 1 / (a * sRes['NbTot Pars'] + b)  # Trop pénalisant: a=0.2, b=1\n",
    "    return 1 / (a * max(2, sRes['NbTot Pars']) + b)  # Mieux: a=0.2, b=0.6\n",
    "\n",
    "def normCVDens(sRes, a=12):\n",
    "    #return max(0, 1 - a * sRes['CoefVar Densité']) # Pas très pénalisant: a=1\n",
    "    return math.exp(-a * sRes['CoefVar Densité'] ** 2) # Mieux : déjà ~0.33 à 30% (a=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combinedGoodnessBalanced(sRes):\n",
    "    return (sRes[['Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P']].prod() \\\n",
    "            * normNObs(sRes) * normNTotPars(sRes, a=0.2, b=0.6) * normCVDens(sRes, a=12)) ** (1.0/7)\n",
    "\n",
    "dfRep['Qual Equi'] = dfRep.apply(combinedGoodnessBalanced, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combinedGoodnessMoreChi2(sRes):\n",
    "    return sRes[['Chi2 P', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P']].prod() \\\n",
    "           * normNObs(sRes) * normNTotPars(sRes, a=0.2, b=0.6) * normCVDens(sRes, a=12) ** (1.0/8)\n",
    "\n",
    "dfRep['Qual Chi2'] = dfRep.apply(combinedGoodnessMoreChi2, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combinedGoodnessMoreKS(sRes):\n",
    "    return sRes[['Chi2 P', 'KS P', 'KS P', 'CvM Uw P', 'CvM Cw P']].prod() \\\n",
    "           * normNObs(sRes) * normNTotPars(sRes, a=0.2, b=0.6) * normCVDens(sRes, a=12) ** (1.0/8)\n",
    "\n",
    "dfRep['Qual KS'] = dfRep.apply(combinedGoodnessMoreKS, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combinedGoodnessMoreDCV(sRes):\n",
    "    return sRes[['Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P']].prod() \\\n",
    "           * normNObs(sRes) * normNTotPars(sRes, a=0.2, b=0.6) \\\n",
    "           * (normCVDens(sRes, a=12) ** 2) ** (1.0/8)\n",
    "\n",
    "dfRep['Qual DCV'] = dfRep.apply(combinedGoodnessMoreDCV, axis='columns')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Groupes de valeurs de distances de troncature\n",
    "\n",
    "Pour pouvoir regrouper les troncatures proches, et ne garder qu'un résultat par groupe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsDist = 1e-6\n",
    "\n",
    "ldTruncIntrvSpecs = [dict(col='Dist Tronc Gche', minDist=5.0, maxLen=5.0),\n",
    "                     dict(col='Dist Tronc Drte', minDist=25.0, maxLen=25.0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pour chaque échantillon,\n",
    "for lblEch, sEch in dfStatsEch.iterrows():\n",
    "    \n",
    "    print(f'#{lblEch}', sEch['Abréviation'], end=': ')\n",
    "\n",
    "    # Pour chaque type de troncature (optimisée ou non),\n",
    "    for isOpt in sorted(dfRep.loc[dfRep.Echant == lblEch, optimTruncCol].unique()):\n",
    "        \n",
    "        print('{}optim'.format('' if isOpt else 'non ').title(), end=' : ')\n",
    "\n",
    "        # Sélectionner les résultats associés, et uniquement ceux-là\n",
    "        dfSelRep = dfRep[(dfRep.Echant == lblEch) & (dfRep[optimTruncCol] == isOpt)]\n",
    "\n",
    "        for dTrunc in ldTruncIntrvSpecs:\n",
    "\n",
    "            truncCol = dTrunc['col']\n",
    "            minIntrvDist = dTrunc['minDist']\n",
    "            maxIntrvLen = dTrunc['maxLen']\n",
    "\n",
    "            print(truncCol, end=', ')\n",
    "\n",
    "            dfIntrv = dfSelRep[[truncCol]].dropna().sort_values(by=truncCol).copy()\n",
    "\n",
    "            # Ecarts non nuls de distances entre distances consécutives triées\n",
    "            dfIntrv['deltaDist'] = dfIntrv[truncCol].diff()\n",
    "            dfIntrv.loc[dfIntrv[truncCol].idxmin(), 'deltaDist'] = np.inf\n",
    "            dfIntrv.dropna(inplace=True)\n",
    "            dfIntrv = dfIntrv[dfIntrv.deltaDist > 0].copy()\n",
    "\n",
    "            # Début et fin de chaque intervalle (fermé à gauche = dMin, ouvert à droite = dSup)\n",
    "            dfIntrv['dMin'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, truncCol]\n",
    "            dfIntrv['dSup'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, truncCol].shift(-1).dropna()\n",
    "            dfIntrv.loc[dfIntrv['dMin'].idxmax(), 'dSup'] = np.inf\n",
    "            dfIntrv.dropna(inplace=True)\n",
    "\n",
    "            sSelDist = dfSelRep[truncCol]\n",
    "            dfIntrv['dSup'] = dfIntrv['dSup'].apply(lambda supV: sSelDist[sSelDist < supV].max() + epsDist)\n",
    "\n",
    "            dfIntrv = dfIntrv[['dMin', 'dSup']].reset_index(drop=True)\n",
    "\n",
    "            # Si les intervalles ainsi détectés sont trop larges, on les découpe en tranches égales\n",
    "            lsNewIntrvs = list()\n",
    "            for _, sIntrv in dfIntrv.iterrows():\n",
    "\n",
    "                if sIntrv.dSup - sIntrv.dMin > maxIntrvLen:\n",
    "                    nSubIntrvs = (sIntrv.dSup - sIntrv.dMin) / maxIntrvLen\n",
    "                    nSubIntrvs = int(nSubIntrvs) if nSubIntrvs - int(nSubIntrvs) < 0.5 else int(nSubIntrvs) + 1\n",
    "                    subIntrvLen = (sIntrv.dSup - sIntrv.dMin) / nSubIntrvs\n",
    "                    lsNewIntrvs += [pd.Series(dict(dMin=sIntrv.dMin + nInd * subIntrvLen, \n",
    "                                                   dSup=min(sIntrv.dMin + (nInd + 1) * subIntrvLen, sIntrv.dSup)))\n",
    "                                    for nInd in range(nSubIntrvs)]\n",
    "                else:\n",
    "                    lsNewIntrvs.append(sIntrv)\n",
    "\n",
    "            dfIntrv = pd.DataFrame(lsNewIntrvs).reset_index(drop=True)\n",
    "            dfIntrv.sort_values(by='dMin', inplace=True)\n",
    "\n",
    "            # Attribution du numéro de groupe de troncatures à chaque distance mesurée (0 = pas de troncature)\n",
    "            dfRep.loc[(dfRep.Echant == lblEch) & (dfRep[optimTruncCol] == isOpt), 'Grp ' + truncCol] = \\\n",
    "                dfSelRep[truncCol].apply(lambda d: 0 if pd.isnull(d) \\\n",
    "                                                   else 1 + dfIntrv[(dfIntrv.dMin <= d) & (dfIntrv.dSup > d)].index[0])\n",
    "\n",
    "        print(len(dfSelRep), end=' ; ')\n",
    "        \n",
    "    print()\n",
    "\n",
    "len(dfRep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. Correction résultats \"Nombre *\" ACDC2019-Nat d'avant le 29/12/2020 (erreur surface zone => facteur 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if repFolder.stem < '201229':\n",
    "    \n",
    "    print(\"Correction des nombres d'individus, suite erreur unité surface zone ACDC 2019.\")\n",
    "    for colNb in ['Nombre', 'Min Nombre', 'Max Nombre']:\n",
    "        dfRep[colNb] /= 100 # ha => km2\n",
    "        \n",
    "    print(dfRep['Nombre'].describe())\n",
    "\n",
    "else:\n",
    "    print('Rien à corriger, suffisamment récent')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Ajout colonnes de filtrage et tri\n",
    "\n",
    "* Ici, on ne supprime pas les lignes, on leur attribue un indice suivant un ordre de tri, global ou par/dans groupe spécifié, \n",
    "* ce qui permet ensuite de filtrer si besoin en ne gardant que les N meilleures lignes (via pandas ou dans un tableur)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spécification des schémas de filtrage / tri\n",
    "filSorSchemes = \\\n",
    "[  # Ordre dans groupe.\n",
    "   dict(name='Meil AIC Tronc Id',  # Meilleur AIC, à troncatures D et G identiques (avec variantes de nb tranches)\n",
    "         sort=['Dist Tronc Gche', 'Dist Tronc Drte',\n",
    "               'Delta AIC', 'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx'],\n",
    "         ascend=[True, True, True, False, False, True, False, True],\n",
    "         group=['Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod']),\n",
    "    \n",
    "   dict(name='Meil CKCv Tronc Proch',  # Meilleur Chi2&KS&DCV par groupe de troncatures proches\n",
    "        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "              'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx'],\n",
    "        ascend=[True, True, True, False, False, True, False, True],\n",
    "        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "#   dict(name='Meil Chi2 Tronc Proch',  # Meilleur Chi2 par groupe de troncatures proches\n",
    "#        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "#              'Chi2 P'],\n",
    "#        ascend=[True, True, True, False],\n",
    "#        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "   dict(name='Meil CVDens Tronc Proch',  # Meilleur DCV par groupe de troncatures proches\n",
    "        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "              'CoefVar Densité'],\n",
    "        ascend=[True, True, True, True],\n",
    "        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "    \n",
    "   dict(name='Meil Qual Equi Tronc Proch',  # Meilleur Qualité combinée équilibrée par groupe de troncatures proches\n",
    "        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "              'Qual Equi'],\n",
    "        ascend=[True, True, True, False],\n",
    "        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "   dict(name='Meil Qual Chi2 Tronc Proch',  # Meilleur Qualité combinée Chi2+ par groupe de troncatures proches\n",
    "        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "              'Qual Chi2'],\n",
    "        ascend=[True, True, True, False],\n",
    "        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "   dict(name='Meil Qual KS Tronc Proch',  # Meilleur Qualité combinée KS+ par groupe de troncatures proches\n",
    "        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "              'Qual KS'],\n",
    "        ascend=[True, True, True, False],\n",
    "        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "   dict(name='Meil Qual DCV Tronc Proch',  # Meilleur Qualité combinée DCV+ par gro  b     \n",
    "        sort=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "              'Qual DCV'],\n",
    "        ascend=[True, True, True, False],\n",
    "        group=[optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']),\n",
    "    \n",
    "   # Ordre global (sans groupage)\n",
    "   dict(name='Ord CKCv',\n",
    "        sort=['Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx'],\n",
    "        ascend=[False, False, True, False, True]),\n",
    "   dict(name='Ord Qual Equi',\n",
    "        sort='Qual Equi', ascend=False),\n",
    "   dict(name='Ord Qual Chi2',\n",
    "        sort='Qual Chi2', ascend=False),\n",
    "   dict(name='Ord Qual KS',\n",
    "        sort='Qual KS', ascend=False),\n",
    "   dict(name='Ord Qual DCV',\n",
    "        sort='Qual DCV', ascend=False),\n",
    "   dict(name='Ord Simpl Tronc',\n",
    "        sort=['Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod',\n",
    "              'Delta AIC', 'Chi2 P', 'KS P', 'CoefVar Densité', 'NObs', 'CodEx'],\n",
    "        ascend=[True, True, True, True, False, False, True, False, True], napos='first'),\n",
    "]\n",
    "\n",
    "# N.B. C'est le dernier ordre qui résulte à la fin ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ajout des colonnes permettant d'appliquer plus tard ces schémas de filtrage / tri\n",
    "ldfEchReps = list()\n",
    "\n",
    "for lblEch, sEch in dfStatsEch.iterrows():\n",
    "\n",
    "    # Sélection des données de l'échantillon\n",
    "    print(f'#{lblEch}', sEch['Abréviation'], end=': ')\n",
    "    dfEchRep = dfRep[dfRep.Echant == lblEch].copy()\n",
    "\n",
    "    # Application des schémas de filtrage / tri\n",
    "    for scheme in filSorSchemes:\n",
    "        dfEchRep.sort_values(by=scheme['sort'], ascending=scheme['ascend'], \n",
    "                             na_position=scheme.get('napos', 'last'), inplace=True)\n",
    "        dfEchRep[scheme['name']] = dfEchRep.groupby(scheme['group'], dropna=False).cumcount() \\\n",
    "                                   if 'group' in scheme else range(len(dfEchRep))\n",
    "\n",
    "    print(len(dfEchRep))\n",
    "    \n",
    "    # Sauvegarde du tableau résultat\n",
    "    ldfEchReps.append(dfEchRep)\n",
    "        \n",
    "# Résultat.\n",
    "dfRep = pd.concat(ldfEchReps) # Keep original report index (row #)\n",
    "\n",
    "len(dfRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export avant filtrage\n",
    "fpn = pl.Path(xlsxRep)\n",
    "fpn = fpn.with_name(fpn.name.replace('rapport', f'rapenrich')) #.with_suffix('.ods')\n",
    "\n",
    "print('=>', fpn.as_posix())\n",
    "\n",
    "dfRep.to_excel(fpn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Filtrage et tri grâce aux colonnes ad hoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterDuplicates(dfRes, keep='first', subset=list(), round2decs=dict()):\n",
    "    \n",
    "    if round2decs:\n",
    "        #dfRes = dfRes.round(round2decs) # Buggy (pandas 1.0.x up to 1.1.2): forgets columns !?!?!?\n",
    "        dfRes = dfRes.copy()\n",
    "        for col, dec in round2decs.items():\n",
    "            if subset and col in subset:  # No useless work !\n",
    "                dfRes[col] = dfRes[col].apply(lambda x: x if pd.isnull(x) else round(x, ndigits=dec))\n",
    "\n",
    "        # Don't use df.round ... because it does not work, at least with pandas 1.0.x up to 1.1.2 !?!?!?\n",
    "        #df = df.round(decimals={ col: dec for col, dec in self.trEnColNames(dColDecimals).items() \\\n",
    "        #                                  if col in df.columns })\n",
    "        \n",
    "    return dfRes[dfRes.duplicated(keep=keep, subset=subset)].index\n",
    "\n",
    "dupSubset = ['Echant', 'NObs', 'Effort', 'Delta AIC',\n",
    "             'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P', 'CoefVar Densité', \n",
    "             'PDetec', 'Min PDetec', 'Max PDetec', 'Densité', 'Min Densité', 'Max Densité']\n",
    "dupRounds = {'Delta AIC': 1, 'Chi2 P': 2, 'KS P': 2, 'CvM Uw P': 2, 'CvM Cw P': 2, 'CoefVar Densité': 2, \n",
    "             'PDetec': 3, 'Min PDetec': 3, 'Max PDetec': 3, 'Densité': 2, 'Min Densité': 2, 'Max Densité': 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterFixScheme(dfRes, sampleIds, sampleIdCol, nMinRes, sbIsToBeDroppped, dfDropThresholds):\n",
    "    \n",
    "    \"\"\"Fonction générique de filtrage avec stratégie de contrôle du nombre de résultats conservé\n",
    "    via un schéma prédéfini de seuillage sur un critère\n",
    "    \"\"\"\n",
    "    \n",
    "    i2Drop = []\n",
    "    for sampId in sampleIds:\n",
    "\n",
    "        dfSampRes = dfRes[dfRes[sampleIdCol] == sampId]\n",
    "        #print('#{} : {}'.format(sampId, len(dfSampRes)), end=' => ')\n",
    "\n",
    "        for _, sThreshold in dfDropThresholds.iterrows():\n",
    "            i2DropSamp = dfSampRes[sbIsToBeDroppped(dfSampRes, nMinRes, **sThreshold)].index\n",
    "            #print(len(i2DropSamp), end=', ')\n",
    "            if len(dfSampRes) - len(i2DropSamp) >= nMinRes:\n",
    "                break\n",
    "\n",
    "        #print(' => ', chi2, len(i2DropSamp))\n",
    "        i2Drop = i2DropSamp if not len(i2Drop) else i2Drop.append(i2DropSamp)\n",
    " \n",
    "    return i2Drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def isToBeDroppedOnChi2(dfRes, nMinRes, chi2):\n",
    "    \n",
    "    return dfRes['Chi2 P'] < chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def isToBeDroppedOnCombinedQuality(dfRes, nMinRes, quality):\n",
    "    \n",
    "    return dfRes['Qualité'] < quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterDichotScheme(dfRes, sampleIds, sampleIdCol, critCol, ascendCrit=True,\n",
    "                       minCritStep=0.001, nMinRes=10, verbose=False):\n",
    "    \n",
    "    \"\"\"Fonction générique de filtrage avec stratégie de contrôle du nombre de résultats conservé\n",
    "    via un schéma adaptatif dichotomique de seuillage sur 1 critère (fonction de son domaine réel de valeurs)\n",
    "    \"\"\"\n",
    "    \n",
    "    # For each sample ...\n",
    "    i2Drop = []\n",
    "    for sampId in sampleIds:\n",
    "\n",
    "        # Extract results.\n",
    "        dfSampRes = dfRes[dfRes[sampleIdCol] == sampId]\n",
    "        if verbose: print('#{}: {} results'.format(sampId, len(dfSampRes)), end=' => ')\n",
    "\n",
    "        # Compute criteria threshold variation scheme from actual value domain\n",
    "        start = dfSampRes[critCol].max() if ascendCrit else dfSampRes[critCol].min()\n",
    "        stop = dfSampRes[critCol].min() if ascendCrit else dfSampRes[critCol].max()\n",
    "        if verbose: print(f'{critCol} [{start:.3f},{stop:.3f}]', end=': ')\n",
    "\n",
    "        # No need for tweeking criteria thresholds, we won't get more results.\n",
    "        if len(dfSampRes) <= nMinRes:\n",
    "            if verbose: print('t={:.3f}/k={}'.format(stop, len(dfSampRes)), end=', ')\n",
    "            if verbose: print('done, no more possible.')\n",
    "            continue\n",
    "        \n",
    "        # For each step of the scheme ...\n",
    "        i2DropSamp, thresh = [], start\n",
    "        while True:\n",
    "            \n",
    "            # Next try : middle of the interval to explore.\n",
    "            threshTry = (start + stop) / 2\n",
    "\n",
    "            # Try and apply the threshold step : number of dropped results if ...\n",
    "            if ascendCrit:\n",
    "                i2DropSampTry = dfSampRes[dfSampRes[critCol] < threshTry].index\n",
    "            else:\n",
    "                i2DropSampTry = dfSampRes[dfSampRes[critCol] > threshTry].index\n",
    "\n",
    "            if verbose: print('t={:.3f}/k={}'.format(threshTry, len(dfSampRes) - len(i2DropSampTry)), end=', ')\n",
    "\n",
    "            # Stop here if the min number expected of results would be reached\n",
    "            if len(dfSampRes) - len(i2DropSampTry) == nMinRes:\n",
    "                i2DropSamp, thresh = i2DropSampTry, threshTry\n",
    "                if verbose: print('done, target reached.')\n",
    "                break\n",
    "                \n",
    "            # Stop when no change in list to drop and above the min number expected of results.\n",
    "            elif len(i2DropSampTry) == len(i2DropSamp) and abs(start - stop) < minCritStep:\n",
    "                if verbose: print('done, no more change.')\n",
    "                break\n",
    "                            \n",
    "            # Update criteria interval to explore according to whether we would be\n",
    "            #  below or above the min number expected of results if ...\n",
    "            if len(dfSampRes) - len(i2DropSampTry) > nMinRes:\n",
    "                if ascendCrit:\n",
    "                    stop = threshTry\n",
    "                else:\n",
    "                    start = threshTry\n",
    "            else:\n",
    "                if ascendCrit:\n",
    "                    start = threshTry\n",
    "                else:\n",
    "                    stop = threshTry\n",
    "                    \n",
    "            # Or else, save current try, and go on.\n",
    "            i2DropSamp, thresh = i2DropSampTry, threshTry\n",
    "\n",
    "        # Append index to drop for sample to the final one\n",
    "        i2Drop = i2DropSamp if not len(i2Drop) else i2Drop.append(i2DropSamp)\n",
    " \n",
    "    return i2Drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "previewCols = ['Echant'] + indexCols \\\n",
    "              + ['Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod', 'CodEx', 'NObs',\n",
    "                 'Delta AIC', 'Chi2 P', 'KS P', 'CoefVar Densité',\n",
    "                 'Sélection Qual Equi',\n",
    "                 'Qual Equi', 'Qual Chi2', 'Qual KS', 'Qual DCV',\n",
    "                 'Densité', 'Min Densité', 'Max Densité']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddfFilSorRep = dict()  # { method: dfFilSorRep}\n",
    "ldFilSorSteps = list()  # [(method, step, param, value)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Méthode \"codexec\"\n",
    "\n",
    "Fitrage minimal et tri :\n",
    "1. Eliminer CodEx 3 et +"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 5\n",
    "preSelCol = 'Qual Equi'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method = 'codexec'\n",
    "\n",
    "print(f'Méthode \"{method}\"')\n",
    "\n",
    "dfFilSorRep = dfRep.copy()\n",
    "ldFilSorSteps.append((method, 'Avant', 'Résultats', len(dfFilSorRep)))\n",
    "print('* Avant :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep.CodEx > 2].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'CodEx', 'Max', 2))\n",
    "ldFilSorSteps.append((method, 'CodEx', 'Résultats', len(dfFilSorRep)))\n",
    "print('* CodEx :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.sort_values(by=['Echant', 'Dist Tronc Gche', 'Dist Tronc Drte', 'CodEx'],\n",
    "                        ascending=True, na_position='first', inplace=True)\n",
    "#dfFilSorRep.drop_duplicates(subset=['Echant', 'NObs', 'Effort', 'Delta AIC',\n",
    "#                                    'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P', 'CoefVar Densité', \n",
    "#                                    'PDetec', 'Min PDetec', 'Max PDetec', 'Densité', 'Min Densité', 'Max Densité'],\n",
    "#                            keep='first', inplace=True)\n",
    "dfFilSorRep.drop(filterDuplicates(dfFilSorRep, keep='first', subset=dupSubset, round2decs=dupRounds),\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Doublons', 'Résultats', len(dfFilSorRep)))\n",
    "print('* Doublons :', len(dfFilSorRep))\n",
    "\n",
    "selPreSelCol = 'Sélection ' + preSelCol\n",
    "dfFilSorRep[selPreSelCol] = dfFilSorRep.groupby(['Echant'] + indexCols)[preSelCol] \\\n",
    "                                       .transform(lambda s: s.rank(ascending=False, method='dense'))\n",
    "dfFilSorRep.loc[dfFilSorRep[selPreSelCol] > nPreSel, selPreSelCol] = np.nan\n",
    "\n",
    "ldFilSorSteps.append((method, 'Pré-sélection auto', 'NbPréSélections', nPreSel))\n",
    "ldFilSorSteps.append((method, 'Pré-sélection auto', 'ColonnePréSélection', preSelCol))\n",
    "print('* Pré-sélection auto: {}{}'.format(nPreSel, preSelCol.replace(' ', '')))\n",
    "\n",
    "dfFilSorRep.sort_values(by=['Echant', 'Dist Tronc Gche', 'Dist Tronc Drte', 'Ord Qual Equi'],\n",
    "                        ascending=True, na_position='first', inplace=True)\n",
    "ldFilSorSteps.append((method, 'Tri', 'Colonnes', 'TroncGche, TroncDrte, QualEqui'))\n",
    "print('* Tri : TroncGche, TroncDrte, QualEqui')\n",
    "\n",
    "ddfFilSorRep[method] = dfFilSorRep\n",
    "\n",
    "dfFilSorRep[previewCols].head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Méthodes \"ckcvN-fN\" (obsolètes) : filtrage sur Chi2&KS&DCV ... adaptatif à seuils\n",
    "\n",
    "Fitrage et tri :\n",
    "1. Eliminer CodEx 3 et +,\n",
    "2. Par groupe de troncatures Gche et Drte et nb tranches fitting identiques,\n",
    "   garder le meilleur AIC (et Chi2 & KS & DCV & CodEx & NObs),\n",
    "3. Par groupe de troncatures optimisées Gche et Drte proches (algo. de groupage à seuils, + gdrs à droite),\n",
    "   garder le meilleur Chi2 & KS & DCV & CodEx & NObs,\n",
    "4. Garder les Taux d'obs conservés >= 95%,\n",
    "5. Garder les Chi2 >= 0.8 (sauf si moins de 5 résultats : baisser le seuil jusqu'à ...),\n",
    "6. Trier par absence / simplicité des troncatures (sans < sans gche < sans drte < avec gche et dte) et CKCv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 3\n",
    "preSelCol = 'Qual Equi'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sightRate = 95\n",
    "nBestAIC = 1\n",
    "nBestCKCv = 1\n",
    "nResults = 5\n",
    "startChi2 = 0.8\n",
    "stopChi2 = 0.1\n",
    "nChi2Steps = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method = f'ckcv{int(sightRate*10)}f{nResults}'\n",
    "\n",
    "print(f'Méthode \"{method}\"')\n",
    "\n",
    "dfFilSorRep = dfRep.copy()\n",
    "ldFilSorSteps.append((method, 'Avant', 'Résultats', len(dfFilSorRep)))\n",
    "print('* Avant :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep.CodEx > 2].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'CodEx', 'Max', 2))\n",
    "ldFilSorSteps.append((method, 'CodEx', 'Résultats', len(dfFilSorRep)))\n",
    "print('* CodEx :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.sort_values(by=['Echant', 'Dist Tronc Gche', 'Dist Tronc Drte', 'CodEx'],\n",
    "                        ascending=True, na_position='first', inplace=True)\n",
    "dfFilSorRep.drop(filterDuplicates(dfFilSorRep, keep='first', subset=dupSubset, round2decs=dupRounds),\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Doublons', 'Résultats', len(dfFilSorRep)))\n",
    "print('* Doublons :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep['Meil AIC Tronc Id'] >= nBestAIC].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'AIC TroncId', 'NbMeilleurs', nBestAIC))\n",
    "ldFilSorSteps.append((method, 'AIC TroncId', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* Meil{nBestAIC}AIC TroncId:', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[(dfFilSorRep[optimTruncCol] == 1)\n",
    "                             & (dfFilSorRep['Meil CKCv Tronc Proch'] >= nBestCKCv)].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'TroncOpt & MeilleursCKCv TroncProch', 'NbMeilleurs', nBestCKCv))\n",
    "ldFilSorSteps.append((method, 'TroncOpt & MeilleursCKCv TroncProch', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* TrOpt & {nBestCKCv}CKCv:', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep['Taux Obs'] < sightRate].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Taux Obs conservées', 'Min', sightRate))\n",
    "ldFilSorSteps.append((method, 'Taux Obs conservées', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* TauxObs{sightRate} :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(filterFixScheme(dfFilSorRep, sampleIds=dfFilSorRep.Echant.unique(), sampleIdCol='Echant',\n",
    "                                 nMinRes=nResults, sbIsToBeDroppped=isToBeDroppedOnChi2,\n",
    "                                 dfDropThresholds=pd.DataFrame(dict(chi2=np.linspace(start=startChi2,\n",
    "                                                                                     stop=stopChi2, num=nChi2Steps)))),\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Meilleurs Chi2', 'Début', startChi2))\n",
    "ldFilSorSteps.append((method, 'Meilleurs Chi2', 'Fin', stopChi2))\n",
    "ldFilSorSteps.append((method, 'Meilleurs Chi2', 'NbPas', nChi2Steps))\n",
    "ldFilSorSteps.append((method, 'Meilleurs Chi2', 'NbCible', nResults))\n",
    "ldFilSorSteps.append((method, 'Meilleurs Chi2', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* {nResults}fChi2 :', len(dfFilSorRep))\n",
    "\n",
    "selPreSelCol = 'Sélection ' + preSelCol\n",
    "dfFilSorRep[selPreSelCol] = dfFilSorRep.groupby(['Echant'] + indexCols)[preSelCol] \\\n",
    "                                       .transform(lambda s: s.rank(ascending=False, method='dense'))\n",
    "dfFilSorRep.loc[dfFilSorRep[selPreSelCol] > nPreSel, selPreSelCol] = np.nan\n",
    "\n",
    "ldFilSorSteps.append((method, 'Pré-sélection auto', 'NbPréSélections', nPreSel))\n",
    "ldFilSorSteps.append((method, 'Pré-sélection auto', 'ColonnePréSélection', preSelCol))\n",
    "print('* Pré-sélection auto: {}{}'.format(nPreSel, preSelCol.replace(' ', '')))\n",
    "\n",
    "dfFilSorRep.sort_values(by=['Echant', 'Dist Tronc Gche', 'Dist Tronc Drte', 'Ord CKCv'],\n",
    "                        ascending=True, na_position='first', inplace=True)\n",
    "ldFilSorSteps.append((method, 'Tri', 'Colonnes', 'TroncGche, TroncDrte, Chi2&KS&...DCV'))\n",
    "print('* Tri : TroncGche, TroncDrte, Chi2&KS...DCV')\n",
    "                     \n",
    "ddfFilSorRep[method] = dfFilSorRep\n",
    "\n",
    "dfFilSorRep[previewCols].head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Méthodes \"ckcvqualN-dN\" : Filtrage sur plusieurs critères, et adaptatif dichotomique\n",
    "\n",
    "Filtrage et tri proche de 1 (?) mais moins méchant, pour action manuelles de filtrage a posteriori\n",
    "1. Eliminer CodEx 3 et +,\n",
    "2. Par groupe de troncatures Gche et Drte et nb tranches fitting identiques,\n",
    "   garder les N1 meilleurs AIC & Chi2 & KS & DCV & NObs & CodEx,\n",
    "3. Par groupe de troncatures Gche et Drte proches (algo. de groupage à seuils, analyses optim / non optim séparées), garder :\n",
    "    * les N2 meilleur Chi2 & KS & DCV & NObs & CodEx,\n",
    "    * les N2 meilleur DCV & Chi2 & KS & NObs & CodEx,\n",
    "    * les N2 meilleur indicateurQualitéCombiné(Chi2, KS, DCV, NObs, CodEx),\n",
    "4. Garder les Taux d'obs conservés >= N3%,\n",
    "5. Garder les N4 meilleurs résultats selon indicateurQualitéCombiné(Chi2, KS, DCV, NObs, CodEx),\n",
    "6. Trier par absence / simplicité des troncatures (sans < sans gche < sans drte < avec gche et dte) et ce même indicateur."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preSelCol = 'Qual Equi'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 3\n",
    "sightRate = 97.5\n",
    "nBestAIC = 2\n",
    "nBestQua = 1\n",
    "nResults = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 3\n",
    "sightRate = 95\n",
    "nBestAIC = 2\n",
    "nBestQua = 1\n",
    "nResults = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 4\n",
    "sightRate = 92.5\n",
    "nBestAIC = 3\n",
    "nBestQua = 1\n",
    "nResults = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 4\n",
    "sightRate = 90\n",
    "nBestAIC = 3\n",
    "nBestQua = 1\n",
    "nResults = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nPreSel = 5\n",
    "sightRate = 90\n",
    "nBestAIC = 4\n",
    "nBestQua = 1\n",
    "nResults = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A exécuter pour chacune des cellules de paramétrage ci-dessus retenues\n",
    "method = f'ckcvqual{int(sightRate*10)}d{nResults}'\n",
    "\n",
    "print(f'Méthode \"{method}\"')\n",
    "\n",
    "dfFilSorRep = dfRep.copy()\n",
    "ldFilSorSteps.append((method, 'Avant', 'Résultats', len(dfFilSorRep)))\n",
    "print('* Avant :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep.CodEx > 2].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'CodEx', 'Max', 2))\n",
    "ldFilSorSteps.append((method, 'CodEx', 'Résultats', len(dfFilSorRep)))\n",
    "print('* CodEx :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.sort_values(by=['Echant', 'Dist Tronc Gche', 'Dist Tronc Drte', 'CodEx'],\n",
    "                        ascending=True, na_position='first', inplace=True)\n",
    "dfFilSorRep.drop(filterDuplicates(dfFilSorRep, keep='first', subset=dupSubset, round2decs=dupRounds),\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Doublons', 'Résultats', len(dfFilSorRep)))\n",
    "print('* Doublons :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep['Meil AIC Tronc Id'] >= nBestAIC].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'AIC TroncId', 'NbMeilleurs', nBestAIC))\n",
    "ldFilSorSteps.append((method, 'AIC TroncId', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* {nBestAIC}AIC TroncId:', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[(dfFilSorRep['Meil CKCv Tronc Proch'] >= nBestQua)\n",
    "                             & (dfFilSorRep['Meil CVDens Tronc Proch'] >= nBestQua)\n",
    "                             & (dfFilSorRep['Meil Qual Equi Tronc Proch'] >= nBestQua)\n",
    "                             & (dfFilSorRep['Meil Qual Chi2 Tronc Proch'] >= nBestQua)\n",
    "                             & (dfFilSorRep['Meil Qual KS Tronc Proch'] >= nBestQua)\n",
    "                             & (dfFilSorRep['Meil Qual DCV Tronc Proch'] >= nBestQua)].index,\n",
    "                           # & (dfFilSorRep['Meil AIC Tronc Id'] > 0)].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'MeilleursCKCv+CVDens+QualEqui+Chi2+KS+DCV TroncProch', 'NbMeilleurs', nBestQua))\n",
    "ldFilSorSteps.append((method, 'MeilleursCKCv+CVDens+QualEqui+Chi2+KS+DCV TroncProch', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* {nBestQua}CKCv+CVDens+QualEqui+Chi2+KS+DCV TroncProc:', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(dfFilSorRep[dfFilSorRep['Taux Obs'] < sightRate].index,\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Taux Obs conservées', 'Min', sightRate))\n",
    "ldFilSorSteps.append((method, 'Taux Obs conservées', 'Résultats', len(dfFilSorRep)))\n",
    "print(f'* TauxObs{sightRate} :', len(dfFilSorRep))\n",
    "\n",
    "dfFilSorRep.drop(filterDichotScheme(dfFilSorRep, sampleIds=dfFilSorRep.Echant.unique(), sampleIdCol='Echant',\n",
    "                                    critCol='Qual Equi', ascendCrit=True, nMinRes=nResults),\n",
    "                 inplace=True)\n",
    "ldFilSorSteps.append((method, 'Meilleurs QualEqui', 'NbCible', nResults))\n",
    "ldFilSorSteps.append((method, 'Meilleurs QualEqui', 'Résultats', len(dfFilSorRep)))\n",
    "\n",
    "print(f'* {nResults}dQual :', len(dfFilSorRep))\n",
    "\n",
    "selPreSelCol = 'Sélection ' + preSelCol\n",
    "dfFilSorRep[selPreSelCol] = dfFilSorRep.groupby(['Echant'] + indexCols)[preSelCol] \\\n",
    "                                       .transform(lambda s: s.rank(ascending=False, method='dense'))\n",
    "dfFilSorRep.loc[dfFilSorRep[selPreSelCol] > nPreSel, selPreSelCol] = np.nan\n",
    "\n",
    "ldFilSorSteps.append((method, 'Pré-sélection auto', 'NbPréSélections', nPreSel))\n",
    "ldFilSorSteps.append((method, 'Pré-sélection auto', 'ColonnePréSélection', preSelCol))\n",
    "print('* Pré-sélection auto: {}{}'.format(nPreSel, preSelCol.replace(' ', '')))\n",
    "\n",
    "dfFilSorRep.sort_values(by=['Echant', 'Dist Tronc Gche', 'Dist Tronc Drte', 'Ord Qual Equi'],\n",
    "                        ascending=True, na_position='first', inplace=True)\n",
    "ldFilSorSteps.append((method, 'Tri', 'Colonnes', 'TroncGche, TroncDrte, QualEqui'))\n",
    "print('* Tri : TroncGche, TroncDrte, QualEqui')\n",
    "\n",
    "ddfFilSorRep[method] = dfFilSorRep\n",
    "\n",
    "dfFilSorRep[previewCols].head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Exports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#expCols1 = ['Analyse', 'Fn Clé Mod', 'Sér Ajust Mod', optimTruncCol,\n",
    "#            'Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod', 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte',\n",
    "#            'CodEx', 'NObs', 'NTot Obs', 'Taux Obs'] \\\n",
    "#           + [scheme['name'] for scheme in filSorSchemes] \\\n",
    "#           + ['Delta AIC', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P', 'Qualité', 'NbTot Pars',\n",
    "#              'CoefVar Densité', 'Densité', 'Min Densité', 'Max Densité',\n",
    "#              'PDetec', 'Min PDetec', 'Max PDetec']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pour valider les méthodes de filtrage\n",
    "#expCols2 = ['Analyse', 'Fn Clé Mod', 'Sér Ajust Mod', 'Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod'] \\\n",
    "#           + ['NObs', 'Delta AIC', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P', 'Taux Obs',\n",
    "#              'CoefVar Densité', 'NbTot Pars', 'Qual Equi', 'Qual Chi2', 'Qual KS', 'Qual DCV'] \\\n",
    "#           + ['Densité', 'Min Densité', 'Max Densité'] \\\n",
    "#           + [scheme['name'] for scheme in filSorSchemes] \\\n",
    "#           + [optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte'] \\\n",
    "#           + ['CodEx', 'NTot Obs'] \\\n",
    "#           + ['PDetec', 'Min PDetec', 'Max PDetec']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pour équivalent rapport (en terme de colonnes)\n",
    "expCols3 = ['Analyse', 'Fn Clé Mod', 'Sér Ajust Mod', 'Dist Tronc Gche', 'Dist Tronc Drte', 'Tranch Dist Mod'] \\\n",
    "           + ['Delta AIC', 'Chi2 P', 'KS P', 'CvM Uw P', 'CvM Cw P', 'NObs', 'Taux Obs', 'CoefVar Densité', 'NbTot Pars'] \\\n",
    "           + ['Sélection finale', 'Sélection Qual Equi'] \\\n",
    "           + ['Qual Equi', 'Qual Chi2', 'Qual KS', 'Qual DCV'] \\\n",
    "           + ['Densité', 'Min Densité', 'Max Densité', 'EDR/ESW', 'Min EDR/ESW', 'Max EDR/ESW',\n",
    "              'Nombre', 'Min Nombre', 'Max Nombre', 'PDetec', 'Min PDetec', 'Max PDetec'] \\\n",
    "           + ['CodEx', 'NTot Obs'] \\\n",
    "           + [scheme['name'] for scheme in filSorSchemes] \\\n",
    "           + [optimTruncCol, 'Grp Dist Tronc Gche', 'Grp Dist Tronc Drte']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expCols = expCols3\n",
    "\n",
    "assert len(expCols) == len(set(expCols)), 'Ho, ho ... some duplicated columns ?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddfFilSorRep.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = ['ckcvqual975d8', 'ckcvqual950d10', 'ckcvqual925d12', 'ckcvqual900d15', 'ckcvqual900d20', 'codexec']\n",
    "\n",
    "assert all(meth in ddfFilSorRep for meth in methods), \\\n",
    "       ','.join(meth for meth in methods if meth not in ddfFilSorRep) + ' not computed'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Historique des étapes, paramètres et résultats des différents filtrages / tris (traçabilité).\n",
    "dfFilSorHist = pd.DataFrame(ldFilSorSteps, columns=['Méthode', 'Etape', 'Variable', 'Valeur']).set_index(['Méthode'])\n",
    "dfFilSorHist = dfFilSorHist.loc[methods]\n",
    "dfFilSorHist.reset_index(inplace=True)\n",
    "dfFilSorHist.set_index(['Méthode', 'Etape'], inplace=True)\n",
    "dfFilSorHist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. 1 seul onglet par méthode, tous les échantillons à chaque fois"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpn = pl.Path(xlsxRep)\n",
    "fpn = fpn.with_name(fpn.name.replace('rapport', f'raptousech')) #.with_suffix('.ods')\n",
    "\n",
    "selectCol = 'Sélection Qual Equi'\n",
    "print('All {} samples:'.format(len(dfStatsEch)))\n",
    "with pd.ExcelWriter(fpn) as xlsWrtr:\n",
    "    for meth in methods:\n",
    "        dfFilSorRep = ddfFilSorRep[meth].copy()\n",
    "        dfFilSorRep.insert(dfFilSorRep.columns.get_loc(selectCol), 'Sélection finale',\n",
    "                           dfFilSorRep[selectCol].where(dfFilSorRep[selectCol] == 1))\n",
    "        dfFilSorRep[['Echant'] + indexCols + expCols].to_excel(xlsWrtr, sheet_name=meth, index=False)\n",
    "        print('* {}: {} results'.format(meth, len(dfFilSorRep)))\n",
    "    dfFilSorHist.to_excel(xlsWrtr, sheet_name='paramètres', index=True)\n",
    "    dfStatsEch.reset_index().to_excel(xlsWrtr, sheet_name='échantillons', index=False)\n",
    "\n",
    "print('=>', fpn.as_posix())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. 1 classeur par méthode, 1 onglet par échantillon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectCol = 'Sélection Qual Equi'\n",
    "for meth in methods:\n",
    "    dfFilSorRep = ddfFilSorRep[meth].copy()\n",
    "    dfFilSorRep.insert(dfFilSorRep.columns.get_loc(selectCol), 'Sélection finale',\n",
    "                       dfFilSorRep[selectCol].where(dfFilSorRep[selectCol] == 1))\n",
    "    fpn = pl.Path(xlsxRep)\n",
    "    fpn = fpn.with_name(fpn.name.replace('rapport', f'raparech-{meth}')) #.with_suffix('.ods')\n",
    "    with pd.ExcelWriter(fpn) as xlsWrtr:\n",
    "        for lblEch, sEch in dfStatsEch.iterrows():\n",
    "            print('* #{} {}'.format(lblEch, sEch['Abréviation']), end=': ')\n",
    "            dfFSEchRep = dfFilSorRep[dfFilSorRep.Echant == lblEch]\n",
    "            dfFSEchRep[expCols].to_excel(xlsWrtr, index=False,\n",
    "                                         sheet_name='{} {}'.format(sEch['Abréviation'], lblEch)) #, engine='odf')\n",
    "            print(len(dfFSEchRep), 'results')\n",
    "        dfFilSorHist.loc[meth].to_excel(xlsWrtr, sheet_name='histoire', index=False)\n",
    "        dfStatsEch.reset_index().to_excel(xlsWrtr, sheet_name='échantillons', index=False)\n",
    "    print('=>', fpn.as_posix())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Exports avec résultats manuels intercalés"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpn = pl.Path(xlsxRep)\n",
    "fpn = fpn.with_name(fpn.name.replace('rapport', f'raptousech-comp')) #.with_suffix('.ods')\n",
    "fpn.as_posix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Chargement et mise en forme résultats manuels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### i. ZPS Crêtes Cantal 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chargement\n",
    "dfManuRep = pd.read_excel(dossier / f'{nomEtude}{sousEtude}-AnalysesD73Mathis-resultats.xlsx')\n",
    "\n",
    "# Nom des colonnes\n",
    "dfManuRep.rename(columns={'Id EchTronc': 'AbrevEch', 'Nb données': 'NObs', 'Modèle': 'Fn Clé Mod',\n",
    "                          'GOF Chi-p': 'Chi2 P', 'D CV': 'CoefVar Densité',\n",
    "                          'D': 'Densité', 'D LCL': 'Min Densité', 'D UCL': 'Max Densité'}, inplace=True)\n",
    "\n",
    "# Colonnes inutiles pour la comparaison\n",
    "dfManuRep.drop(columns=['N', 'N LCL', 'N UCL', 'SURF HAB FAVORABLE', 'D / 10 ha', 'Remarques'], inplace=True)\n",
    "\n",
    "# Suppression des lignes sans intérêt (commentaires)\n",
    "dfManuRep.dropna(subset=['AbrevEch'], inplace=True)\n",
    "\n",
    "# Conversions diverses\n",
    "for col in ['Densité', 'Min Densité', 'Max Densité']:\n",
    "    dfManuRep[col] /= 100\n",
    "\n",
    "# Identification de l'échantillon : à partir de l'id. de l'analyse\n",
    "dfManuRep.AbrevEch = dfManuRep.AbrevEch.apply(lambda s: '-'.join(s.split('-')[:2]))\n",
    "\n",
    "# Colonne pour identifier la source\n",
    "dfManuRep.insert(0, 'Source', 0)\n",
    "\n",
    "# Colonne pour pouvoir conserver l'ordre de tri initial\n",
    "dfManuRep.insert(0, 'Ordre', range(len(dfManuRep)))\n",
    "\n",
    "# Colonne numéro d'échantillon.\n",
    "dfManuRep = dfManuRep.join(dfStatsEch[['Abréviation']].reset_index().set_index('Abréviation'), on='AbrevEch')\n",
    "\n",
    "dfManuRep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. 1 seul onglet par méthode, tous les échantillons à chaque fois"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpn = pl.Path(xlsxRep)\n",
    "fpn = fpn.with_name(fpn.name.replace('rapport', f'raptousech-comp')) #.with_suffix('.ods')\n",
    "\n",
    "print('All {} samples ({} manual results):'.format(len(dfStatsEch), len(dfManuRep)))\n",
    "\n",
    "with pd.ExcelWriter(fpn) as xlsWrtr:\n",
    "    \n",
    "    for meth in methods:\n",
    "        \n",
    "        dfFilSorRep = ddfFilSorRep[meth].copy()\n",
    "        \n",
    "        dfFilSorRep.insert(1, 'AbrevEch', dfFilSorRep[['Espèce', 'Adulte']].apply(sampleAbbrev, axis='columns'))\n",
    "        dfFilSorRep.insert(0, 'Source', 1)\n",
    "        dfFilSorRep.insert(0, 'Ordre', range(len(dfFilSorRep)))\n",
    "        nAutoRes = len(dfFilSorRep)\n",
    "        \n",
    "        dfFilSorRep = dfFilSorRep.append(dfManuRep, ignore_index=True)\n",
    "        \n",
    "        dfFilSorRep.sort_values(by=['Echant', 'Source', 'Ordre'], ascending=True, inplace=True)\n",
    "        \n",
    "        dfFilSorRep[['Source', 'Echant'] + indexCols + expCols].to_excel(xlsWrtr, sheet_name=meth, index=False) #, engine='odf')\n",
    "        \n",
    "        print('* {}: {} results (auto: {})'.format(meth, len(dfFilSorRep), nAutoRes))\n",
    "\n",
    "    dfFilSorHist.to_excel(xlsWrtr, sheet_name='histoire', index=True)\n",
    "\n",
    "print('=>', fpn.as_posix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Rapports Excel et HTML\n",
    "\n",
    "Pré-requis :\n",
    "* export filtré prêt pour chargement (1 à 5 ci-dessus) => fichier <etude>-raptousech.ods\n",
    "* résultats d'optanalyses produits ou chargés (XVI.2a/b) => variable results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rechargement de l'export filtré\n",
    "fpn = pl.Path(xlsxRep)\n",
    "fpn = fpn.with_name(fpn.name.replace('rapport', f'raptousech')).with_suffix('.ods')\n",
    "print(fpn.as_posix())\n",
    "\n",
    "ddfFilSorExp = pd.read_excel(fpn, sheet_name=None)\n",
    "ddfFilSorExp.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# On travaille avec la méthode 'ckcvqual925d12'\n",
    "selectCol = 'Sélection Qual Equi'\n",
    "selectMeth = 'ckcvqual925d12'\n",
    "\n",
    "dfFilSorExp = ddfFilSorExp[selectMeth]\n",
    "selAnlysIds = dfFilSorExp[dfFilSorExp[selectCol].notnull()].Analyse.tolist()\n",
    "print(dict(ciblees=len(selAnlysIds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Résultats pour le rapport = résultats d'opt-analyse, après filtrage : Supprimer les analyses hors 'Sélection Qual Equi'\n",
    "filSorRes = results.copy()\n",
    "sResAnalysIds = filSorRes.dfData[('header (head)', 'Analyse', 'Value')]\n",
    "print(dict(optanalyses=len(sResAnalysIds), ciblees=sResAnalysIds.isin(selAnlysIds).sum()))\n",
    "     \n",
    "filSorRes.dropRows(~sResAnalysIds.isin(selAnlysIds))\n",
    "print(dict(filtrees=len(filSorRes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Complément des résultats pour rapport : colonnes 'Qual Equi'\n",
    "dfFilSorRes2Join = dfFilSorExp[['Analyse', 'Qual Equi']]\n",
    "dfFilSorRes2Join.columns = pd.MultiIndex.from_tuples([('header', 'Analyse', 'Value'),\n",
    "                                                      ('filtering', 'Qual Equi', 'Value')])\n",
    "dfFilSorRes2Join.set_index(('header', 'Analyse', 'Value'), inplace=True)\n",
    "\n",
    "# Forcer le calcul des colonnes ... calculées, si pas déjà fait.\n",
    "_ = filSorRes.dfData\n",
    "\n",
    "# Bricolage : les données\n",
    "filSorRes._dfData = filSorRes._dfData.join(dfFilSorRes2Join, on=[('header (head)', 'Analyse', 'Value')])\n",
    "\n",
    "# Bricolage : les traductions (nom colonnes)\n",
    "filSorRes.dfCustomColTrans.loc[('filtering', 'Qual Equi', 'Value')] = pd.Series(dict(en='Bal. Quality', fr='Qualité Equi.'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filSorRes.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sélection des colonnes pour les tableaux du pré-rapport\n",
    "# a. Page principale : Colonne 1 (haut), de description de l'échantillon\n",
    "filSorRepSampleCols = \\\n",
    "[('header (head)', 'Echant', 'Value')] \\\n",
    "+ [('header (sample)', col, 'Value') for col in samplingCols] \\\n",
    "+ [('header (head)', 'Analyse', 'Value')]\n",
    "\n",
    "# b. Page principale : Colonne 1 (bas), des paramètres du modèle d'analyse\n",
    "filSorRepParamCols = \\\n",
    "[\n",
    "    ('parameters', 'estimator key function', 'Value'),\n",
    "    ('parameters', 'estimator adjustment series', 'Value'),\n",
    "    #('parameters', 'CV interval', 'Value')\n",
    "    ('parameters', 'left truncation distance', 'Value'),\n",
    "    ('parameters', 'right truncation distance', 'Value'),\n",
    "    ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "]\n",
    "\n",
    "# c. Page principale : Colonne 2 et 3, des résultats (juste avant les 4, 5, et 6 avec les graphiques)\n",
    "filSorRepResultCols = \\\n",
    "[\n",
    "    ('run output', 'run status', 'Value'),\n",
    "    \n",
    "    ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "    ('encounter rate', 'right truncation distance (w)', 'Value'),\n",
    "    ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "    \n",
    "    #('detection probability', 'AIC value', 'Value'),\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "    ('detection probability', 'number of adjustment term parameters (NAP)', 'Value'),\n",
    "    ('filtering', 'Qual Equi', 'Value'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "\n",
    "    ('density/abundance', 'density of animals', 'Cv'),\n",
    "    ('density/abundance', 'density of animals', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Lcl'),\n",
    "    ('density/abundance', 'density of animals', 'Ucl'),\n",
    "    \n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Value'),\n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Lcl'),\n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Ucl')\n",
    "]\n",
    "\n",
    "# d. Pages ppale et de détails : Tableau de synthése.\n",
    "filSorRepSynthCols = filSorRepSampleCols + filSorRepParamCols \\\n",
    "+ [\n",
    "    ('run output', 'run status', 'Value'),\n",
    "    \n",
    "    ('encounter rate', 'number of observations (n)', 'Value'),\n",
    "    ('encounter rate', 'right truncation distance (w)', 'Value'),\n",
    "    ('encounter rate', 'effort (L or K or T)', 'Value'),\n",
    "    ('encounter rate', 'encounter rate (n/L or n/K or n/T)', 'Df'),\n",
    "    \n",
    "    ('detection probability', 'AIC value', 'Value'),\n",
    "    ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "    ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Cv'),\n",
    "    \n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Lcl'),\n",
    "    ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Ucl'),\n",
    "    \n",
    "    ('density/abundance', 'density of animals', 'Value'),\n",
    "    ('density/abundance', 'density of animals', 'Lcl'),\n",
    "    ('density/abundance', 'density of animals', 'Ucl'),\n",
    "    \n",
    "    ('detection probability', 'probability of detection (Pw)', 'Value'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Lcl'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Ucl'),\n",
    "    ('detection probability', 'probability of detection (Pw)', 'Df'),\n",
    "\n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Value'),\n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Lcl'),\n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Ucl'),\n",
    "    ('density/abundance', 'number of animals, if survey area is specified', 'Df'),\n",
    "   \n",
    "    ('run output', 'run folder', 'Value')\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filSorRepSortCols = \\\n",
    "[('header (head)', sampleNumCol, 'Value')] \\\n",
    "+ [('filtering', 'Qual Equi', 'Value'),\n",
    "   ('detection probability', 'chi-square test probability determined', 'Value'),\n",
    "   ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value')\n",
    "]\n",
    "\n",
    "filSorRepAscend = [True] + [False]*3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filSorRep = ads.MCDSResultsFullReport(resultsSet=filSorRes, title=titreEtude,\n",
    "                                      subTitle=f\"Rapport d'analyse (après filtrage / sélection '{selectMeth}')\",\n",
    "                                      anlysSubTitle='Détail des analyses filtrées', description=descrEtude,\n",
    "                                      keywords=motsClesEtude, pySources=['Visionature-ds-points.ipynb'],\n",
    "                                      lang='fr', superSynthPlotsHeight=288, plotImgSize=(640, 400),\n",
    "                                      #plotImgQuality=80, plotImgFormat='jpg', # Same final size as raw PNG :-(\n",
    "                                      sampleCols=filSorRepSampleCols, paramCols=filSorRepParamCols,\n",
    "                                      resultCols=filSorRepResultCols, synthCols=filSorRepSynthCols,\n",
    "                                      sortCols=filSorRepSortCols, sortAscend=filSorRepAscend,\n",
    "                                      tgtFolder=workDir, \n",
    "                                      tgtPrefix=f'{nomEtude}{sousEtude}-AnalysesFiltrees-{selectMeth}-rapport')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlsxFilSorRep = filSorRep.toExcel()\n",
    "\n",
    "HTML(f'Rapport Excel : <a href=\"{xlsxFilSorRep}\" target=\"blank\">{xlsxFilSorRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# 4-HT-core i5-8365U PCI-e SSD: 6 generators Naturalist (2021-02-14): 3mn15-20s (n=3)\n",
    "# 4-HT-core i5-8365U PCI-e SSD: 6 generators Naturalist+Papyrus (2021-02-27): s (n=1)\n",
    "htmlFilSorRep = filSorRep.toHtml(generators=6)\n",
    "\n",
    "HTML(f'Pré-rapport HTML : <a href=\"{htmlFilSorRep}\" target=\"blank\">{htmlFilSorRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls = ads.MCDSTruncOptanalysisResultsSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = results._dfData.copy()\n",
    "df2 = results._dfData.copy()\n",
    "df3 = results._dfData.copy()\n",
    "len(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nBestQua = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.drop(df1[(df1[cls.CLGrpOrdClTrChi2KSDCv] >= nBestQua)\n",
    "                             & (df1[cls.CLGrpOrdClTrDCv] >= nBestQua)\n",
    "                             & (df1[cls.CLGrpOrdClTrQuaBal1] >= nBestQua)\n",
    "                             & (df1[cls.CLGrpOrdClTrQuaChi2] >= nBestQua)\n",
    "                             & (df1[cls.CLGrpOrdClTrQuaKS] >= nBestQua)\n",
    "                             & (df1[cls.CLGrpOrdClTrQuaDCv] >= nBestQua)].index,\n",
    "                           # & (df1[cls.CLGrpOrdClTrChi2] > 0)].index,\n",
    "                 inplace=True)\n",
    "len(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "R = cls\n",
    "whichBestQua = [R.CLGrpOrdClTrChi2KSDCv, R.CLGrpOrdClTrDCv, R.CLGrpOrdClTrQuaBal1,\n",
    "                R.CLGrpOrdClTrQuaChi2, R.CLGrpOrdClTrQuaKS, R.CLGrpOrdClTrQuaDCv]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for clQuaIndic in whichBestQua:\n",
    "    df2.drop(df2[df2[clQuaIndic] >= nBestQua].index, inplace=True)\n",
    "    print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sb2drop = None\n",
    "for clQuaIndic in whichBestQua:\n",
    "    sb2dropIndic = (df3[clQuaIndic] >= nBestQua)\n",
    "    if sb2drop is None:\n",
    "        sb2drop = sb2dropIndic\n",
    "    else:\n",
    "        sb2drop &= sb2dropIndic\n",
    "df3.drop(df3[sb2drop].index, inplace=True)\n",
    "print(len(df3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4 = results._dfData.copy()\n",
    "sb2keep = pd.Series(data=False, index=df4.index)\n",
    "for clQuaIndic in whichBestQua:\n",
    "    sb2keep |= (df4[clQuaIndic] < nBestQua)\n",
    "df4 = df4[sb2keep]\n",
    "print(len(df4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4.compare(df3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4.drop(df4[df4[cls.CLSightRate] < 92.5].index, inplace=True)\n",
    "len(df4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.DataFrame([dict(a=True, b=True), dict(a=False, b=True), dict(a=True, b=False), dict(a=False, b=False)])\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.drop(df1[(df1.a == True) & (df1.b == True)].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([dict(a=1), dict(a=2)])\n",
    "a = np.array([[2, 3], [4, 5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['b', 'c']] = a\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py38]",
   "language": "python",
   "name": "conda-env-py38-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
