{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 class='tocIgnore'>ACDC 2019 Naturalist: Distance Sampling analyses with pyaudisam</h1>\n",
    "\n",
    "(on a reduced data sample)\n",
    "\n",
    "Please read first [how-it-works-fr.md](../how-it-works/how-it-works-fr.md)\n",
    "\n",
    "<!-- Auto table of contents -->\n",
    "<div style=\"overflow-y: auto\">\n",
    "  <h2 class='tocIgnore'>Table of content</h2>\n",
    "  <div id=\"toc\"></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "// Generate TOC in above cell\n",
    "var maxlevel = 3;\n",
    "$.getScript('ipython_notebook_toc.js', function() {createTOC(maxlevel);})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "<!- Left align markdown tables in cells -->\n",
    "<style> table { display: inline-block }</style>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: This notebook was developped and tested under the following platform:\n",
    "\n",
    "|            |                 |\n",
    "|:-----------|:-----------------|\n",
    "| os         | Windows Enterprise 10.0.19044 (64bit) |\n",
    "| processor  | Intel64 Family 6 Model 165 Stepping 2, GenuineIntel, 12 CPUs |\n",
    "| python     | cpython (win32) R3.8.15, packaged by conda-forge, (default, Nov 22 2022, 08:43:00), MSC v.1929 64 bits (AMD64) |\n",
    "| numpy      | 1.23.2 |\n",
    "| pandas     | 1.2.5 |\n",
    "| zoopt      | 0.4.0 |\n",
    "| matplotlib | 3.4.2 |\n",
    "| jinja2     | 3.0.1 |\n",
    "| pyaudisam  | 1.0.0 |\n",
    "| DS engine  | C:/PortableApps/Distance 7.3/MCDS.exe |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import shutil\n",
    "import pathlib as pl\n",
    "import re\n",
    "import datetime as dt\n",
    "\n",
    "import pandas as pd\n",
    "import pandas.api.types as pdt\n",
    "import numpy as np\n",
    "\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comment-out to use site-packages installed pyaudisam version.\n",
    "sys.path.insert(0, '../..')  # Or not ... to use local source / dev. package one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyaudisam as ads\n",
    "\n",
    "print('pyaudisam', ads.__version__, 'from', pl.Path(ads.__path__[0]).resolve().as_posix())\n",
    "\n",
    "ads.runtime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Commons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create temporary directory if not yet done.\n",
    "tmpDir = pl.Path('../../tmp')\n",
    "tmpDir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logging configuration.\n",
    "ads.log.configure(handlers=[sys.stdout, tmpDir / 'acdc2019.log'], reset=True,\n",
    "                  loggers=[dict(name='matplotlib', level=ads.WARNING),\n",
    "                           dict(name='ads', level=ads.INFO2),\n",
    "                           #dict(name='ads.eng', level=ads.INFO),\n",
    "                           #dict(name='ads.exr', level=ads.INFO),\n",
    "                           #dict(name='ads.anr', level=ads.DEBUG1),\n",
    "                           #dict(name='ads.onr', level=ads.DEBUG),\n",
    "                           dict(name='acdc2019', level=ads.DEBUG)])\n",
    "\n",
    "logger = ads.logger('acdc2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backup(fpn, to='.', tsFmt='.%y%m%d-%H%M%S'):\n",
    "    \"\"\"Backup given file to target folder with custom-formatted timestamp in name\"\"\"\n",
    "    fpn = pl.Path(fpn)\n",
    "    tn = fpn.stem + pd.Timestamp.now().strftime(tsFmt) + fpn.suffix\n",
    "    tp = pl.Path(to) if to != '.' else fpn.parent\n",
    "    logger.info('Backing up to ' + (tp / tn).as_posix())\n",
    "    shutil.copy(fpn, tp / tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Define study parameters\n",
    "\n",
    "Note: You can run this cell whenever you have changed something in `acdc-2019-ds-params.py`, no need to restart the kernel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load parameter file (the same that is usable through pyaudisam command line)\n",
    "parFile, pars = ads.loadPythonData('./acdc-2019-nat-ds-params.py')\n",
    "assert pars, f'Failed to load parameter file {parFile.as_posix()}'\n",
    "\n",
    "pars  # A types.SimpleNamespace instance (use dot = '.' to access to parameters by name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# II. Load individualised observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with pd.ExcelFile(pars.surveyDataFile) as xlsFile:\n",
    "    dfIndDistObs = pd.read_excel(xlsFile, sheet_name=pars.indivDistDataSheet)\n",
    "    dfTransects = pd.read_excel(xlsFile, sheet_name=pars.transectsDataSheet)\n",
    "\n",
    "print(dict(study=pars.studyName+pars.subStudyName, observations=len(dfIndDistObs), transects=len(dfTransects)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTransects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfIndDistObs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [V. Automated pre-analyses / 1b. Or: Load pre-analysis results from a previous session](#1b.-Or%3A-Load-pre-analysis-results-from-a-previous-session),\n",
    "* [VI. Automated (opt-)analyses](#VI.-Automated-(opt-)analyses),\n",
    "* [VI. Automated (opt-)analyses / 2b. Or: Load (opt-)analyses results from a previous session](#2b.-Or%3A-Load--(opt-)analyses-results-from-a-previous-session),\n",
    "* [Appendix. Sumup and stats for indivisualised sightings](#Appendix.-Sumup-and-stats-for-individualised-sightings),\n",
    "\n",
    "or simply to next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# III. Selection of samples for Distance export and pre-analyses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Field data examination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfIndDistObs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of individuals per species, in order to decide which we'll DS-analyse\n",
    "if pars.clustering: # Clustering of individuals (1 observation may be about multiple individuals)\n",
    "    \n",
    "    # Warning: Not tested yet.\n",
    "    dfNObsCatIndiv = dfIndDistObs[pars.sampleSelCols + ['Nombre']].groupby(pars.sampleSelCols).sum()\n",
    "    dfNObsCatIndiv.rename(columns=dict(Nombre='Individus'), inplace=True)\n",
    "    \n",
    "else: # Individuals taken 1 by 1.\n",
    "    \n",
    "    dfNObsCatIndiv = dfIndDistObs[pars.sampleSelCols + [pars.distanceCol]].groupby(pars.sampleSelCols).count()\n",
    "    dfNObsCatIndiv.rename(columns=dict(Distance='Individus'), inplace=True)\n",
    "\n",
    "dfNObsCatIndiv.reset_index(inplace=True)\n",
    "\n",
    "dfNObsCatIndiv.sort_values(by='Individus', ascending=False).head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Specify samples to analyse: select variants for all category column\n",
    "\n",
    "* \"Espèce\" = Species,\n",
    "* \"Passage\" = Visits (for inventory) on transect points (a = early speing, b = late spring, a+b = both),\n",
    "* \"Durée\" = Inventory duration (5mn or 10mn),\n",
    "* \"Adulte\" = Population type (m = adult males only, a = other adults, unsexed males or females)\n",
    "\n",
    "**Warning** : the \"Passage\" column must be there for pre-analyses and analyses, even if there's only one pass.\n",
    "\n",
    "Some other english translations ;-)\n",
    "* \"Indivu\" = individual = 1 bird\n",
    "* "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. First possible method: Specify variants through a dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Initialisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs = dict()  # Key = Category names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some stats to help decide: Number of individuals per inventory duration for each species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfNObsIndiv = dfNObsCatIndiv[['Espèce', 'Durée', 'Individus']].groupby(['Espèce', 'Durée']).sum().unstack() \\\n",
    "                .sort_values(by=('Individus', '10mn'), ascending=False)\n",
    "dfNObsIndiv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or : Species with at least N individuals observed over 10mn field inventories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nMinTotIndivs = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs[pars.speciesCol] = list(dfNObsIndiv[dfNObsIndiv[('Individus', '10mn')] >= nMinTotIndivs].index)\n",
    "print(len(sampleSpecs[pars.speciesCol]), ', '.join(sampleSpecs[pars.speciesCol]))\n",
    "\n",
    "sampleSpecs[pars.passIdCol] = [''] # All passes together"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or :The N most numerous species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nMostNumerous = 3\n",
    "\n",
    "sampleSpecs[pars.speciesCol] = list(dfNObsIndiv[('Individus', '10mn')].index[:nMostNumerous])\n",
    "print(len(sampleSpecs[pars.speciesCol]), ', '.join(sampleSpecs[pars.speciesCol]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some other stats to help decide: Number of males per inventory duration for each species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfNObsMale = dfNObsCatIndiv.loc[dfNObsCatIndiv['Adulte'] == 'm', ['Espèce', 'Durée', 'Individus']] \\\n",
    "                .groupby(['Espèce', 'Durée']).sum().unstack() \\\n",
    "                .sort_values(by=('Individus', '10mn'), ascending=False)\n",
    "dfNObsMale"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or : The species with at least N males observed during the 10mn field inventories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nMinMal10 = 35\n",
    "\n",
    "sampleSpecs[pars.speciesCol] = list(dfNObsMale[dfNObsMale[('Individus', '10mn')] >= nMinMal10].index)\n",
    "print(', '.join(sampleSpecs[pars.speciesCol]), '=>', len(sampleSpecs[pars.speciesCol]), 'species')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Or: Explicit list of species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs[pars.speciesCol] = ['Sylvia atricapilla', 'Prunella modularis', 'Phylloscopus bonelli', 'Oriolus oriolus']\n",
    "print(len(sampleSpecs[pars.speciesCol]), 'species =>', ', '.join(sampleSpecs[pars.speciesCol]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mandatory: Add variant specs for pass, duration and population type categories\n",
    "\n",
    "(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs[pars.passIdCol] = ['b', 'a+b'] # Passes b or a+b => 2 variants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs['Durée'] = ['5mn', '10mn'] # 5 first mn, or all 10 mn => 2 variants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs['Adulte'] = ['m', 'm+a'] # Males, and then males + other adults => 2 variants"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Finally: All of the above variant specs are implicit ones\n",
    "\n",
    "... so, let's assert it !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs = dict(_impl=sampleSpecs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Second possible method: Specify variants through a workbook\n",
    "\n",
    "(with possibly multiple sheets, each possibly being of 'implict' kind, or of 'explicit' kind)\n",
    "\n",
    "TODO: Write a documentation for the way the parser for that all works ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleSpecs = pars.sampleSpecFile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [IV. Export data for manual analyses through Distance software GUI](#IV.-Export-data-for-manual-analyses-through-Distance-software-GUI),\n",
    "* [V. Automated pre-analyses](#V.-Automated-pre-analyses),\n",
    "* [VI. Automated (opt-)analyses](#VI.-Automated-(opt-)analyses),\n",
    "\n",
    "or simply to next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.Option:  For info, some stats about thus specified samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Once explicitated, here are all the selected variants ...\n",
    "dfExplSampleSpecs = ads.DSAnalyser.explicitVariantSpecs(sampleSpecs)\n",
    "dfExplSampleSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Max distance and nb of individuals for each sample, in the analysis order\n",
    "# a. Real sample selection columns (some might be empty)\n",
    "dfSampleOrd = dfExplSampleSpecs.dropna(axis='columns', how='all')\n",
    "indexCols = [col for col in pars.sampleSelCols if col in dfSampleOrd.columns]\n",
    "\n",
    "# b. Order of species in sample list\n",
    "dfSampleOrd = dfSampleOrd.drop_duplicates(subset=[pars.speciesCol])[[pars.speciesCol]].reset_index(drop=True)\n",
    "dfSampleOrd = dfSampleOrd.reset_index(drop=False).rename(columns=dict(index='order'))\n",
    "dfSampleOrd = dfSampleOrd.set_index(pars.speciesCol)\n",
    "\n",
    "# c. Max distances for each unique value of each category (ex: Adulte => m, a)\n",
    "dfSampleStats = dfIndDistObs[indexCols + [pars.distanceCol]].groupby(indexCols).agg(['min', 'max', 'count'])\n",
    "dfSampleStats.columns = ['Min Distance', 'Max Distance', 'NTot Obs']\n",
    "dfSampleStats = dfSampleStats.reset_index()\n",
    "\n",
    "# d. Categories for combination + simple sorting\n",
    "speciesOrder = list(dfSampleOrd.index) + [e for e in dfSampleStats[pars.speciesCol].unique() if e not in dfSampleOrd.index]\n",
    "\n",
    "dCategories = {pars.speciesCol: speciesOrder, pars.passIdCol: ['a', 'b', 'a+b'],\n",
    "               'Adulte': ['m', 'a', 'm+a'], 'Durée': ['5mn', '10mn']}\n",
    "dCategoryTypes = { cat: pdt.CategoricalDtype(categories=values, ordered=True) for cat, values in dCategories.items()}\n",
    "\n",
    "for col in indexCols:\n",
    "    dfSampleStats[col] = dfSampleStats[col].astype(dCategoryTypes[col])\n",
    "\n",
    "# e. Max distances for combined values of non-nested categories (ex: Adulte => m+a)\n",
    "allNestedCategories = {'Durée': '10mn'}\n",
    "nonSpeciesSampleSelCols = [col for col in pars.sampleSelCols if col != pars.speciesCol]\n",
    "cols2OrCombine = [col for col in nonSpeciesSampleSelCols if col in indexCols and col not in allNestedCategories]\n",
    "for col2OrComb in cols2OrCombine:\n",
    "    indexNoCol2CombCols = [col for col in indexCols if col != col2OrComb]\n",
    "    dfSampleStatsOrComb = \\\n",
    "        dfSampleStats[indexNoCol2CombCols + ['Min Distance', 'Max Distance', 'NTot Obs']].groupby(indexNoCol2CombCols) \\\n",
    "            .agg({'Min Distance': 'min', 'Max Distance': 'max', 'NTot Obs': 'sum'})\n",
    "    dfSampleStatsOrComb.columns = ['Min Distance', 'Max Distance', 'NTot Obs']\n",
    "    dfSampleStatsOrComb = dfSampleStatsOrComb.dropna().reset_index()  # Why Nans appear in index ? A mystery !\n",
    "    dfSampleStatsOrComb[col2OrComb] = '+'.join(dfSampleStats[col2OrComb].sort_values().unique())\n",
    "    dfSampleStatsOrComb[col2OrComb] = dfSampleStatsOrComb[col2OrComb].astype(dCategoryTypes[col2OrComb])\n",
    "    dfSampleStats = dfSampleStats.append(dfSampleStatsOrComb, ignore_index=True)\n",
    "    \n",
    "# d. Tri dans l'ordre des espèces, et des autres colonnes de sélection d'échantillon    \n",
    "dfSampleStats.sort_values(by=indexCols, inplace=True)  # Magic ! (thanks to CategoricalDtype)\n",
    "\n",
    "dfSampleStats.reset_index(inplace=True, drop=True)\n",
    "\n",
    "dfSampleStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save these stats (not needed for computations below, though).\n",
    "fpn = tmpDir / f'{pars.studyName}{pars.subStudyName}-SampleStats.xlsx'\n",
    "\n",
    "dfSampleStats.to_excel(fpn, index=False)\n",
    "\n",
    "logger.info(fpn.as_posix())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [IV. Export data for manual analyses through Distance software GUI](#IV.-Export-data-for-manual-analyses-through-Distance-software-GUI),\n",
    "* [V. Automated pre-analyses](#V.-Automated-pre-analyses),\n",
    "* [VI. Automated (opt-)analyses](#VI.-Automated-(opt-)analyses),\n",
    "\n",
    "or simply to next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IV. Export data for manual analyses through Distance software GUI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Output folder for exported files\n",
    "workDir = tmpDir / dt.datetime.now().strftime('%y%m%d-%H%M%S')\n",
    "workDir.as_posix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a PreAnalyser instance (it knows how to export :-).\n",
    "pranlysr = ads.MCDSPreAnalyser(dfIndDistObs, dfTransects=dfTransects, effortConstVal=pars.passEffort,\n",
    "                               dSurveyArea=pars.studyAreaSpecs, transectPlaceCols=pars.transectPlaceCols,\n",
    "                               passIdCol=pars.passIdCol, effortCol=pars.effortCol,\n",
    "                               sampleSelCols=pars.sampleSelCols, sampleDecCols=[pars.effortCol, pars.distanceCol],\n",
    "                               sampleIndCol=pars.sampleIndCol,\n",
    "                               abbrevCol=pars.sampleAbbrevCol, abbrevBuilder=pars.sampleAbbrev,\n",
    "                               distanceUnit=pars.distanceUnit, areaUnit=pars.areaUnit,\n",
    "                               surveyType=pars.surveyType, distanceType=pars.distanceType,\n",
    "                               clustering=pars.clustering,\n",
    "                               workDir=workDir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export data for the selected samples.\n",
    "pranlysr.exportDSInputData(implSampleSpecs=sampleSpecs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# V. Automated pre-analyses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1a. Or: Really run the pre-analyses\n",
    "\n",
    "Prerequisites: run notebook at least up to end of [III. Selection of samples for Distance Sampling analyses](#III.-Selection-of-samples-for-Distance-Sampling-analyses)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Output folder for results, reports, ...\n",
    "workDir = tmpDir / dt.datetime.now().strftime('%y%m%d-%H%M%S')\n",
    "\n",
    "# Output result workbook file.\n",
    "preResFileNameSufx = 'PreAnalyses-results'\n",
    "presFileName = workDir / f'{pars.studyName}{pars.subStudyName}-{preResFileNameSufx}.xlsx'\n",
    "\n",
    "presFileName.as_posix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a pre-analyser object.\n",
    "pranlysr = ads.MCDSPreAnalyser(dfIndDistObs, dfTransects=dfTransects, effortConstVal=pars.passEffort,\n",
    "                               dSurveyArea=pars.studyAreaSpecs, transectPlaceCols=pars.transectPlaceCols,\n",
    "                               passIdCol=pars.passIdCol, effortCol=pars.effortCol,\n",
    "                               sampleSelCols=pars.sampleSelCols, sampleDecCols=[pars.effortCol, pars.distanceCol],\n",
    "                               sampleIndCol=pars.sampleIndCol,\n",
    "                               abbrevCol=pars.sampleAbbrevCol, abbrevBuilder=pars.sampleAbbrev,\n",
    "                               distanceUnit=pars.distanceUnit, areaUnit=pars.areaUnit,\n",
    "                               surveyType=pars.surveyType, distanceType=pars.distanceType,\n",
    "                               clustering=pars.clustering,\n",
    "                               resultsHeadCols=pars.preResultsHeadCols,\n",
    "                               workDir=workDir,\n",
    "                               runMethod=pars.runPreAnalysisMethod, runTimeOut=pars.runPreAnalysisTimeOut,\n",
    "                               logData=pars.logPreAnalysisData, logProgressEvery=pars.logPreAnalysisProgressEvery)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check pre-analysis parameters = sample specs.\n",
    "dfExplSampleSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    pranlysr.explicitParamSpecs(implParamSpecs=sampleSpecs, dropDupes=True, check=True)  \n",
    "\n",
    "logger.info(dict(nSamples=len(dfExplSampleSpecs)))\n",
    "\n",
    "assert userParamSpecCols == [] # No analysis params here (auto. generated by PreAnalyser)\n",
    "assert intParamSpecCols == [] # Idem\n",
    "assert verdict\n",
    "assert not reasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Run pre-analysis, on selected samples, with given model fall-back strategy\n",
    "# Note: Here, we use at most 12-worker parallelism, but you might have to lower this if your PC has less hyper-threaded cores ...\n",
    "preResults = pranlysr.run(implSampleSpecs=sampleSpecs, dModelStrategy=pars.modelPreStrategy, threads=12)\n",
    "\n",
    "# Done.\n",
    "preAnalysed = True\n",
    "\n",
    "pranlysr.shutdown()\n",
    "\n",
    "# Add some more stats to thre results object\n",
    "if 'dfSampleStats' in vars():\n",
    "    preResults.updateSpecs(sampleStats=dfSampleStats)\n",
    "\n",
    "# Save results (might be useful for building a pre-analysis report in a later notebook session)\n",
    "preResults.toExcel(presFileName)\n",
    "\n",
    "backup(presFileName)  # Just in case you wanna keep results from multiple tries with diffferent samples specs or strategies ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Performances figures on a 6-core HT i7-10850H Ruindows 10 (2023) laptop with PCI-e SSD, \"optimal performance power scheme\", Python 3.8.15: **between 0.5 and 1.5s per sample with 12 threads** (depending on the analysis failure rate and model strategy)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [2. Build pre-analysis Excel and/or HTML report(s)](#2.-Build-pre-analysis-Excel-and%2For-HTML-report(s))."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1b. Or: Load pre-analysis results from a previous session\n",
    "\n",
    "Prerequisites: run notebook at least up to end of [II. Load individualised observations](#II.-Load-individualised-observations)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'preAnalysed' not in vars():\n",
    "    preAnalysed = False  # No, we've got pre-results just generated in this session, so we'll use them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not preAnalysed:\n",
    "    \n",
    "    if 'preResFileNameSufx' not in vars():\n",
    "        preResFileNameSufx = 'PreAnalyses-results'\n",
    "    \n",
    "    # List tmpDir sub-folders that contain pre-analysis results\n",
    "    resFolders = [fn.name for fn in tmpDir.glob('[0-9]'*6+'-'+'[0-9]'*6)\n",
    "                  if (fn / f'{pars.studyName}{pars.subStudyName}-{preResFileNameSufx}.xlsx').is_file()]\n",
    "    \n",
    "    logger.info('Available pre-analysis results:')\n",
    "    for index, folder in enumerate(resFolders):\n",
    "        logger.info(f'[{index}] {folder}')\n",
    "    \n",
    "    # Choose manually the desired folder\n",
    "    preResDirIndex = int(input(f'Enter the integer index of the chosen folder, in [0, {len(resFolders) - 1}]: '))\n",
    "    workDir = tmpDir / resFolders[preResDirIndex]\n",
    "    \n",
    "    presFileName = workDir / f'{pars.studyName}{pars.subStudyName}-{preResFileNameSufx}.xlsx'\n",
    "    \n",
    "    logger.info(f'Selected result file: {presFileName}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not preAnalysed:\n",
    "    \n",
    "    # An analyser object knowns how to build an empty results object ...\n",
    "    # But beware: We have to use the same constructor parameters as for the instance used to generate these results !\n",
    "    pranlysr = ads.MCDSPreAnalyser(dfIndDistObs, dfTransects=dfTransects, effortConstVal=pars.passEffort,\n",
    "                                   dSurveyArea=pars.studyAreaSpecs, transectPlaceCols=pars.transectPlaceCols,\n",
    "                                   passIdCol=pars.passIdCol, effortCol=pars.effortCol,\n",
    "                                   sampleSelCols=pars.sampleSelCols, sampleDecCols=[pars.effortCol, pars.distanceCol],\n",
    "                                   sampleIndCol=pars.sampleIndCol,\n",
    "                                   abbrevCol=pars.sampleAbbrevCol, abbrevBuilder=pars.sampleAbbrev,\n",
    "                                   distanceUnit=pars.distanceUnit, areaUnit=pars.areaUnit,\n",
    "                                   surveyType=pars.surveyType, distanceType=pars.distanceType,\n",
    "                                   clustering=pars.clustering,\n",
    "                                   resultsHeadCols=pars.preResultsHeadCols)\n",
    "    \n",
    "    preResults = pranlysr.setupResults()\n",
    "    \n",
    "    # Load results from file\n",
    "    preResults.fromFile(presFileName)\n",
    "    \n",
    "else:\n",
    "    \n",
    "    logger.info('Pre-analyses just run, results are still in kernel memory: no need to reload.')\n",
    "    \n",
    "logger.info('... {} pre-analyses ready for reporting'.format(len(preResults)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Build pre-analysis Excel and/or HTML report(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preResults.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a pre-analysis report object.\n",
    "preReport = ads.MCDSResultsPreReport(resultsSet=preResults,\n",
    "                                     title=pars.preReportStudyTitle, subTitle=pars.preReportStudySubTitle,\n",
    "                                     anlysSubTitle=pars.preReportAnlysSubTitle, description=pars.preReportStudyDescr,\n",
    "                                     keywords=pars.reportStudyKeywords, lang=pars.studyLang, \n",
    "                                     pySources=['acdc-2019-nat-ds-run.ipynb', 'acdc-2019-nat-ds-params.py'],\n",
    "                                     sampleCols=pars.preReportSampleCols, paramCols=pars.preReportParamCols,\n",
    "                                     resultCols=pars.preReportResultCols, synthCols=pars.preReportSynthCols,\n",
    "                                     sortCols=pars.preReportSortCols, sortAscend=pars.preReportSortAscend,\n",
    "                                     tgtPrefix=f'{pars.studyName}{pars.subStudyName}-preanalyses-report',\n",
    "                                     tgtFolder=workDir, **pars.preReportPlotParams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate the Excel workbook report (.xlsx)\n",
    "xlsxPreRep = preReport.toExcel()\n",
    "\n",
    "backup(xlsxPreRep)  # Just in case you wanna keep reports from multiple tries with diffferent samples specs or strategies ...\n",
    "\n",
    "HTML(f'Excel report: <a href=\"{xlsxPreRep}\" target=\"blank\">{pl.Path(xlsxPreRep).as_posix()}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate the OpenDoc report (.ods)\n",
    "# Note: New feature of Pandas 1.1, but no cell-coloring support through styles yet (as of 1.1.3) :-(\n",
    "\n",
    "# odsPreRep = preReport.toOpenDoc()\n",
    "\n",
    "# backup(odsPreRep)  # Just in case you wanna keep reports from multiple tries with diffferent analysis specs or strategies ...\n",
    "\n",
    "# HTML(f'Rapport OpenDoc/Spreadsheet : <a href=\"{odsPreRep}\" target=\"blank\">{odsPreRep}</a>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Generate the HTML report (using 6 parallel generators: consider lowering this figure on less powerfull computers).\n",
    "htmlPreRep = preReport.toHtml(generators=6)\n",
    "\n",
    "backup(htmlPreRep)  # Just in case you wanna keep reports from multiple tries with diffferent samples specs or strategies ...\n",
    "                    # But warning here: it's HTML, and linked files in analysis-specific subfolders are not backup !\n",
    "\n",
    "HTML(f'HTML pre-report: <a href=\"{htmlPreRep}\" target=\"blank\">{pl.Path(htmlPreRep).as_posix()}</a>')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Performances figures on a 6-core HT i7-10850H Ruindows 10 (2023) laptop with PCI-e SSD, \"optimal performance power scheme\", Python 3.8.15: around **1.5s per sample with 6 parallel generators**."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [VI. Automated (opt-)analyses / 2b. Or: Load (opt-)analyses results from a previous session](#2b.-Or%3A-Load--(opt-)analyses-results-from-a-previous-session),\n",
    "\n",
    "or simply to next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VI. Automated (opt-)analyses\n",
    "\n",
    "(with possible auto-determination of truncation distance parameters, through an optimisation technique)\n",
    "\n",
    "Prerequisites: run notebook at least up to end of [III. Selection of samples for Distance Sampling analyses](#III.-Selection-of-samples-for-Distance-Sampling-analyses)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Define analysis specs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Or: All analysis variants specified in this workbook file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "studyVariant = ''\n",
    "anlysSpecFileName = f'{pars.studyName}-OptAnalysesToDo'\n",
    "ignoreSpecs = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Or: Idem, but without analysis variants having auto-optimised distance truncation parameters\n",
    "\n",
    "Warning: For this special case, you'll need to remove `rs.CLParModFitDistCuts` from `fullReportParamCols` (respectively `filsorReportParamCols`) in `acdc-2019-ds-params.py` if you want to generate a full report (respectively an auto-filtered and sorted report) ; because after running the analyses, the 'model fitting distance cut points' column is lacking from the results, as there's no more any variant on it after removing the 'AutoTruncations' variant sheet from the spec. workbook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "studyVariant = '-nooptim'\n",
    "anlysSpecFileName = f'{pars.studyName}-OptAnalysesToDo'\n",
    "ignoreSpecs = ['AutoTruncations_impl']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Or : All analysis variants specified in this workbook file, but not re-running the optimisations\n",
    "\n",
    "* first, you need in-memory results, through an opt-analysis run, or through reloading them via [2b. Or: Reload  (opt-)analyses results from a previous session](#2b.-Or%3A-Reload--(opt-)analyses-results-from-a-previous-session) below)\n",
    "* then, you just need to extract the sample id. and analysis parameters, with ready-to-go truncation params (auto-optimised or user-set)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "studyVariant = '-noreoptim'\n",
    "anlysSpecFileName = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfExplOptAnlysSpecs = results.dfTransData(pars.studyLang)[['Espèce', 'Passage', 'Adulte', 'Durée', 'FonctionClé', 'SérieAjust',\n",
    "                                                           'TrGche', 'TrDrte', 'NbTrchMod', 'OptimTrunc']]\n",
    "dfExplOptAnlysSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mandatory: Final checks and preparation, after specifying variants though a spec file or a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict(studyVar=studyVariant, specs=anlysSpecFileName if anlysSpecFileName else 'dfExplOptAnlysSpecs', ignore=ignoreSpecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis specs :\n",
    "# * or: all-implicit ones, from optAnlysSpecs (with possibly some to remove),\n",
    "# * or: all-explicit, from given dfExplOptAnlysSpecs DataFrame.\n",
    "if anlysSpecFileName:\n",
    "    \n",
    "    # No explicit specs from dfExplOptAnlysSpecs.\n",
    "    dfExplOptAnlysSpecs = None\n",
    "    \n",
    "    # All-implicit specs, from the file.\n",
    "    optAnlysSpecFileExts = ['.ods', '.xlsx']\n",
    "    for ext in optAnlysSpecFileExts:\n",
    "        optAnlysSpecs = pars.dataDir / f'{anlysSpecFileName}{ext}'\n",
    "        if optAnlysSpecs.is_file():\n",
    "            break\n",
    "\n",
    "    assert optAnlysSpecs.is_file(), \\\n",
    "           '{} not found, neither those with other extensions [{}] !' \\\n",
    "           .format(optAnlysSpecs.as_posix(), ', '.join(optAnlysSpecFileExts[:-1]))\n",
    "\n",
    "    logger.info('All-implicit specs, through ' + optAnlysSpecs.as_posix())\n",
    "\n",
    "    # Finally, remove some partial specs if specified\n",
    "    if ignoreSpecs:\n",
    "        optAnlysSpecs = pd.read_excel(optAnlysSpecs, sheet_name=None)\n",
    "        for spec in ignoreSpecs:\n",
    "            del optAnlysSpecs[spec]\n",
    "        logger.info('... after removing ' + str(ignoreSpecs))\n",
    "        logger.info('... leaving at the end ' + str(list(optAnlysSpecs.keys())))\n",
    "        \n",
    "else:\n",
    "    \n",
    "    # No implicit specs from a file.\n",
    "    optAnlysSpecs = None\n",
    "    \n",
    "    # All-explicit specs, from given dfExplOptAnlysSpecs.\n",
    "    assert not dfExplOptAnlysSpecs.empty\n",
    "    \n",
    "    logger.info('All-explicit specs, through dfExplOptAnlysSpecs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2a. Or: Really run the (opt-)analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set to True for recovering from where interrupted during optimisations (in order not to redo all from start).\n",
    "recoverOptims = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if recoverOptims:\n",
    "    \n",
    "    # List possible folders for recovery.\n",
    "    logger.info('Available result folders for recovery:')\n",
    "    \n",
    "    bkupFileNamePat = 'optr-resbak-[01].pickle.xz'\n",
    "    resFolders = list()\n",
    "    folderInd = 0\n",
    "    for fpn in sorted(tmpDir.glob('[0-9]'*6+'-'+'[0-9]'*6)):\n",
    "        \n",
    "        bkupFilePathNames = list(fpn.glob(bkupFileNamePat))\n",
    "        if bkupFilePathNames:\n",
    "            \n",
    "            logger.info(f'  [{folderInd}] {fpn.name}')\n",
    "            for bfpn in bkupFilePathNames:\n",
    "                logger.info('    {} {}'.format(bfpn.name, pd.Timestamp.fromtimestamp(bfpn.stat().st_mtime)))\n",
    "            \n",
    "            resFolders.append(fpn.name)\n",
    "            folderInd += 1\n",
    "            \n",
    "    # If any, let the user select the right one.\n",
    "    if folderInd:\n",
    "        \n",
    "        # Choose manually the folder to recover from / go on working inside.\n",
    "        resDirIndex = int(input(f'Enter the integer index of the chosen folder, in [0, {len(resFolders) - 1}]: '))\n",
    "        resFolder = resFolders[resDirIndex]\n",
    "        \n",
    "    # Otherwise, nothing more to achieve.\n",
    "    else:\n",
    "        logger.info('None found, no possible recovery: seems all needed analyses were finally run !')\n",
    "        \n",
    "else:\n",
    "    \n",
    "    # A brand new one.\n",
    "    resFolder = dt.datetime.now().strftime('%y%m%d-%H%M%S')\n",
    "    \n",
    "# Output folder for results, reports ... etc.\n",
    "workDir = tmpDir / resFolder\n",
    "\n",
    "logger.info(f'Selected recovery run folder: {workDir.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Result file\n",
    "resFileNameSufx = f'OptAnalyses{studyVariant}-results'\n",
    "resFileName = workDir / f'{pars.studyName}{pars.subStudyName}-{resFileNameSufx}.xlsx'\n",
    "\n",
    "resFileName.as_posix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the opt-analyser object.\n",
    "optanlr = \\\n",
    "    ads.MCDSTruncationOptanalyser(dfIndDistObs, dfTransects=dfTransects, effortConstVal=pars.passEffort,\n",
    "                                  dSurveyArea=pars.studyAreaSpecs, transectPlaceCols=pars.transectPlaceCols,\n",
    "                                  passIdCol=pars.passIdCol, effortCol=pars.effortCol,\n",
    "                                  sampleSelCols=pars.sampleSelCols, sampleDecCols=[pars.effortCol, pars.distanceCol],\n",
    "                                  sampleDistCol=pars.distanceCol,\n",
    "                                  abbrevCol=pars.analysisAbbrevCol, abbrevBuilder=pars.analysisAbbrev,\n",
    "                                  anlysIndCol=pars.analysisIndCol, sampleIndCol=pars.sampleIndCol,\n",
    "                                  distanceUnit=pars.distanceUnit, areaUnit=pars.areaUnit,\n",
    "                                  surveyType=pars.surveyType, distanceType=pars.distanceType,\n",
    "                                  clustering=pars.clustering,\n",
    "                                  resultsHeadCols=dict(before=[pars.analysisIndCol, pars.sampleIndCol],\n",
    "                                                       sample=pars.sampleSelCols,\n",
    "                                                       after=pars.analysisParamCols + [pars.analysisAbbrevCol]),\n",
    "                                  ldTruncIntrvSpecs=pars.ldTruncIntrvSpecs, truncIntrvEpsilon=pars.truncIntrvEpsilon,\n",
    "                                  workDir=workDir, logData=pars.logOptAnalysisData,\n",
    "                                  runMethod=pars.runOptAnalysisMethod, runTimeOut=pars.runOptAnalysisTimeOut,\n",
    "                                  logAnlysProgressEvery=pars.logOptAnalysisProgressEvery,\n",
    "                                  logOptimProgressEvery=pars.logOptimisationProgressEvery,\n",
    "                                  backupOptimEvery=pars.backupOptimisationsEvery,\n",
    "                                  defEstimKeyFn=pars.defEstimKeyFn, defEstimAdjustFn=pars.defEstimAdjustFn,\n",
    "                                  defEstimCriterion=pars.defEstimCriterion, defCVInterval=pars.defCVInterval,\n",
    "                                  defExpr2Optimise=pars.defExpr2Optimise, defMinimiseExpr=pars.defMinimiseExpr,\n",
    "                                  defOutliersMethod=pars.defOutliersMethod, defOutliersQuantCutPct=pars.defOutliersQuantCutPct,\n",
    "                                  defFitDistCutsFctr=pars.defFitDistCutsFctr, defDiscrDistCutsFctr=pars.defDiscrDistCutsFctr,\n",
    "                                  defSubmitTimes=pars.defSubmitTimes, defSubmitOnlyBest=pars.defSubmitOnlyBest,\n",
    "                                  dDefSubmitOtherParams=pars.dDefSubmitOtherParams,\n",
    "                                  dDefOptimCoreParams=dict(core=pars.defCoreEngine, maxIters=pars.defCoreMaxIters,\n",
    "                                                           termExprValue=pars.defCoreTermExprValue,\n",
    "                                                           algorithm=pars.defCoreAlgorithm, maxRetries=pars.defCoreMaxRetries))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check opt-analysis specs.\n",
    "dfExplOptAnlysSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    optanlr.explicitParamSpecs(implParamSpecs=optAnlysSpecs, dfExplParamSpecs=dfExplOptAnlysSpecs, dropDupes=True, check=True)  \n",
    "\n",
    "if not verdict:\n",
    "    logger.info('Opt-analysis specs errors:')\n",
    "    logger.info('\\n'.join(reasons))\n",
    "else:\n",
    "    logger.info('Opt-analysis specs OK')\n",
    "\n",
    "logger.info(dict(specs=', '.join(optAnlysSpecs.keys()) if isinstance(optAnlysSpecs, dict)\n",
    "                       else optAnlysSpecs.as_posix() if optAnlysSpecs else 'Explicit',\n",
    "                 nOptAnalyses=len(dfExplOptAnlysSpecs), userParamSpecCols=', '.join(userParamSpecCols),\n",
    "                 intParamSpecCols=', '.join(intParamSpecCols), unmUserParamSpecCols=', '.join(unmUserParamSpecCols)))\n",
    "\n",
    "assert verdict\n",
    "assert not reasons\n",
    "\n",
    "# Once explicitated, here are all the selected (opt-)analysis variants ...\n",
    "#dfExplOptAnlysSpecs.to_excel(tmpDir / f'{pars.studyName}{pars.subStudyName}-ExplOptAnlysSpecs.xlsx')\n",
    "\n",
    "dfExplOptAnlysSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If some pre-analyses were run in this session, check that all samples targetted by opt-analyses have been pre-analysed.\n",
    "if 'dfExplSampleSpecs' in vars():\n",
    "    \n",
    "    dfOptAnlysSpecsCheck = dfExplOptAnlysSpecs[pars.sampleSelCols].drop_duplicates()\n",
    "    dfOptAnlysSpecsCheck['OptAnalyses'] = True\n",
    "    dfOptAnlysSpecsCheck.set_index(pars.sampleSelCols, inplace=True)\n",
    "\n",
    "    dfPreAnlysSpecsCheck = dfExplSampleSpecs.copy()\n",
    "    dfPreAnlysSpecsCheck['PreAnalyses'] = True\n",
    "    dfPreAnlysSpecsCheck.set_index(pars.sampleSelCols, inplace=True)\n",
    "    \n",
    "    dfCheck = dfOptAnlysSpecsCheck.join(dfPreAnlysSpecsCheck, how='outer').reset_index()\n",
    "    \n",
    "    logger.info(f'{len(dfCheck)} total samples already pre-analysed or to be analysed ...')\n",
    "    if dfCheck.PreAnalyses.isnull().sum():\n",
    "        logger.info('... and these are the ones that have not been pre-analysed :')\n",
    "        display(dfCheck[dfCheck.PreAnalyses.isnull()])\n",
    "    else:\n",
    "        logger.info('... and all of them have been pre-analysed (good) !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's use implicit specs if provided (default) ... or force using explicitated ones (if we like).\n",
    "\n",
    "# Comment-out to force using explicit specs.\n",
    "#optAnlysSpecs = None\n",
    "\n",
    "# Use implicit specs if given (not deduced explicit ones).\n",
    "if optAnlysSpecs:\n",
    "    dfExplOptAnlysSpecs = None  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Last quick checks about what's gonna be run and how.\n",
    "dict(recoverOptims=recoverOptims, dfExplOptAnlysSpecs=len([] if dfExplOptAnlysSpecs is None else dfExplOptAnlysSpecs),\n",
    "     optAnlysSpecs=optAnlysSpecs if optAnlysSpecs else None, workDir=workDir.as_posix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Run all specified (opt-)analyses.\n",
    "results = optanlr.run(dfExplParamSpecs=dfExplOptAnlysSpecs, implParamSpecs=optAnlysSpecs,\n",
    "                      recoverOptims=recoverOptims, threads=12)\n",
    "\n",
    "# Tip: For running only a subset of the specified variants ...\n",
    "# results = optanlr.run(dfExplOptAnlysSpecs.iloc[0:2], threads=2)\n",
    "\n",
    "# Done.\n",
    "optAnalysed = True\n",
    "\n",
    "optanlr.shutdown()\n",
    "\n",
    "# Add some more stats to the result object\n",
    "if 'dfSampleStats' in vars():\n",
    "    results.updateSpecs(sampleStats=dfSampleStats)\n",
    "\n",
    "# Save results to disk\n",
    "results.toExcel(resFileName)\n",
    "\n",
    "backup(resFileName) # Just in case you wanna keep reports from multiple tries with diffferent analysis specs or samples ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Performances figures on a 6-core HT i7-10850H Ruindows 10 (2023) laptop with PCI-e SSD, \"optimal performance power scheme\", Python 3.8.15: **between 15 and 20 analysis per second** with **12 threads** (1 analysis here means 1 MCDS.exe run: beware, when using optimised trunction parameters, this means a lot of analysis, not only as many as len(dfExplOptAnlysSpecs))."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfData.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [3. Auto-filtered Excel & HTML reports](#3.-Auto-filtered-Excel-%26-HTML-reports),\n",
    "* [4. Full (unfiltered) Excel & HTML reports](#4.-Full-(unfiltered)-Excel-%26-HTML-reports)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2b. Or: Load  (opt-)analyses results from a previous session\n",
    "\n",
    "Prerequisites: run notebook at least up to end of [II. Load individualised observations](#II.-Load-individualised-observations)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'optAnalysed' not in vars():\n",
    "    optAnalysed = False\n",
    "    \n",
    "if not optAnalysed:\n",
    "    \n",
    "    # List tmpDir sub-folders that contain (opt-)analysis results, and retrieve associated study variant (assuming only 1 = 1st)\n",
    "    resFileNameSufx = 'OptAnalyses*-results'\n",
    "    resFolders = {}\n",
    "    for folder in tmpDir.glob('[0-9]'*6+'-'+'[0-9]'*6):\n",
    "        files = list(folder.glob(f'{pars.studyName}{pars.subStudyName}-{resFileNameSufx}.xlsx'))\n",
    "        print(folder, len(files))\n",
    "        for file in files:\n",
    "            mo = re.match(f'{pars.studyName}{pars.subStudyName}-{resFileNameSufx}.xlsx'.replace('*', '(.*)'), file.name)\n",
    "            studyVariant = mo.group(1)\n",
    "            resFolders[folder.name] = studyVariant\n",
    "            print(file, ':', studyVariant)\n",
    "            break\n",
    "            \n",
    "    logger.info('Available (opt-)analyses result folders (with study variant):')\n",
    "    for index, (folder, variant) in enumerate(resFolders.items()):\n",
    "        logger.info(f'[{index}] {folder} : variant=\"{variant}\"')\n",
    "\n",
    "    # Choose manually the desired folder\n",
    "    resDirIndex = int(input(f'Enter the integer index of the chosen folder, in [0, {len(resFolders) - 1}]: '))\n",
    "    resFolder, studyVariant = list(resFolders.items())[resDirIndex]\n",
    "    workDir = tmpDir / resFolder\n",
    "    \n",
    "    resFileNameSufx = f'OptAnalyses{studyVariant}-results'\n",
    "    resFileName = workDir / f'{pars.studyName}{pars.subStudyName}-{resFileNameSufx}.xlsx'\n",
    "    assert resFileName.is_file()\n",
    "    \n",
    "    logger.info(f'Selected result file: {resFileName.as_posix()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not optAnalysed:\n",
    "    \n",
    "    # An opt-analyser object knowns how to build an empty results object ...\n",
    "    # But beware: We have to use the same constructor parameters as for the instance used to generate these results !\n",
    "    optanlr = \\\n",
    "        ads.MCDSTruncationOptanalyser(dfIndDistObs, dfTransects=dfTransects,\n",
    "                                      effortConstVal=pars.passEffort, dSurveyArea=pars.studyAreaSpecs, \n",
    "                                      transectPlaceCols=pars.transectPlaceCols, passIdCol=pars.passIdCol,\n",
    "                                      effortCol=pars.effortCol, sampleSelCols=pars.sampleSelCols,\n",
    "                                      sampleDecCols=[pars.effortCol, pars.distanceCol], sampleDistCol=pars.distanceCol,\n",
    "                                      abbrevCol=pars.analysisAbbrevCol, abbrevBuilder=pars.analysisAbbrev,\n",
    "                                      anlysIndCol=pars.analysisIndCol, sampleIndCol=pars.sampleIndCol,\n",
    "                                      distanceUnit=pars.distanceUnit, areaUnit=pars.areaUnit,\n",
    "                                      surveyType=pars.surveyType, distanceType=pars.distanceType,\n",
    "                                      clustering=pars.clustering,\n",
    "                                      ldTruncIntrvSpecs=pars.ldTruncIntrvSpecs, truncIntrvEpsilon=pars.truncIntrvEpsilon,\n",
    "                                      resultsHeadCols=dict(before=[pars.analysisIndCol, pars.sampleIndCol],\n",
    "                                                           sample=pars.sampleSelCols,\n",
    "                                                           after=pars.analysisParamCols + [pars.analysisAbbrevCol]))\n",
    "    \n",
    "    results = optanlr.setupResults()\n",
    "    \n",
    "    # Load results from file\n",
    "    results.fromFile(resFileName)\n",
    "    \n",
    "else:\n",
    "\n",
    "    logger.info('(Opt-)Analyses just run, results are still in kernel memory: no need to reload.')\n",
    "    \n",
    "logger.info('... {} opt-analyses ready for reporting'.format(len(results)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, you can go straight to:\n",
    "* [4. Full (unfiltered) Excel & HTML reports](#4.-Full-(unfiltered)-Excel-%26-HTML-reports),\n",
    "\n",
    "or simply to next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Auto-filtered Excel & HTML reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an \"auto-filter-and-sort report\" object\n",
    "filSorReport = ads.MCDSResultsFilterSortReport(resultsSet=results,\n",
    "                                               title=pars.optAnlysFilsorReportStudyTitle,\n",
    "                                               subTitle=pars.optAnlysFilsorReportStudySubTitle,\n",
    "                                               anlysSubTitle=pars.optAnlysFilsorReportAnlysSubTitle,\n",
    "                                               description=pars.optAnlysFilsorReportStudyDescr,                                               \n",
    "                                               keywords=pars.optAnlysFilsorReportStudyKeywords, lang=pars.studyLang,\n",
    "                                               pySources=['acdc-2019-nat-ds-run.ipynb', 'acdc-2019-nat-ds-params.py'],\n",
    "                                               sampleCols=pars.filsorReportSampleCols, paramCols=pars.filsorReportParamCols,\n",
    "                                               resultCols=pars.filsorReportResultCols, synthCols=pars.filsorReportSynthCols,\n",
    "                                               sortCols=pars.filsorReportSortCols, sortAscend=pars.filsorReportSortAscend,\n",
    "                                               filSorSchemes=pars.filsorReportSchemes, \n",
    "                                               tgtFolder=workDir,\n",
    "                                               tgtPrefix=f'{pars.studyName}{pars.subStudyName}'\n",
    "                                                         f'-optanalyses{studyVariant}-report',\n",
    "                                               **pars.filsorReportPlotParams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Generate the workbook report (for all the filter & sort schemes available: see pars.filSorRepSchemes)\n",
    "xlsxFilSorRep = filSorReport.toExcel(rebuild=pars.filsorReportRebuild)\n",
    "\n",
    "backup(xlsxFilSorRep)  # Just in case you wanna keep reports from multiple tries with different analysis specs or samples ...\n",
    "\n",
    "logger.info('Excel auto-filtered report: ' + pl.Path(xlsxFilSorRep).as_posix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tip: Open your report workbook in your .xlsx app.\n",
    "os.startfile(xlsxFilSorRep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "\n",
    "# Generate the OpenDoc report (.ods)\n",
    "# Note: New feature of Pandas 1.1, but no cell-coloring support through styles yet (as of 1.1.3) :-(\n",
    "\n",
    "# odsFilSorRep = filsorReport.toOpenDoc(rebuild=pars.filsorReportRebuild)\n",
    "\n",
    "# backup(odsFilSorRep)  # Just in case you wanna keep reports from multiple tries with diffferent analysis specs or samples ...\n",
    "\n",
    "# logger.info('OpenDoc/Workbook auto-filtered report: ' + pl.Path(odsFilSorRep).resolve().as_uri())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Select the filter & sort scheme from the avalable ones (see pars.filsorReportSchemes).\n",
    "htmlFilSorScheme = next(schm for schm in pars.filsorReportSchemes\n",
    "                        if schm['method'] is ads.MCDSTruncOptanalysisResultsSet.filterSortOnExCAicMulQua\n",
    "                           and schm['filterSort']['sightRate'] == 92.5)\n",
    "\n",
    "# Generate the HTML report for the selected filter scheme\n",
    "# Note: No parallelism used here (something to be improved in pyaudisam ;-)\n",
    "htmlFilSorRep = filSorReport.toHtml(htmlFilSorScheme, rebuild=pars.filsorReportRebuild)\n",
    "\n",
    "backup(htmlFilSorRep)  # Just in case you wanna keep reports from multiple tries with diffferent analysis specs or samples ...\n",
    "                       # But warning here: it's HTML, and linked files in analysis-specific subfolders are not backup !\n",
    "\n",
    "afsId = results.filSorSchemeId(htmlFilSorScheme)\n",
    "logger.info(f'HTML auto-filtered report ({afsId} scheme):\\n=> ' + pl.Path(htmlFilSorRep).resolve().as_uri())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Performances figures on a 6-core HT i7-10850H Ruindows 10 (2023) laptop with PCI-e SSD, \"optimal performance power scheme\", Python 3.8.15: **around 2 retained analysis per second** (**no parallel generators** here, because it is disabled got this kind of report, because of some unresolved bug)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Full (unfiltered) Excel & HTML reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a \"full report\" object\n",
    "report = ads.MCDSResultsFullReport(resultsSet=results, \n",
    "                                   title=pars.optAnlysFullReportStudyTitle, subTitle=pars.optAnlysFullReportStudySubTitle,\n",
    "                                   anlysSubTitle=pars.optAnlysFullReportAnlysSubTitle,\n",
    "                                   description=pars.optAnlysFullReportStudyDescr,\n",
    "                                   keywords=pars.optAnlysFullReportStudyKeywords, lang=pars.studyLang,\n",
    "                                   pySources=['acdc-2019-nat-ds-run.ipynb', 'acdc-2019-nat-ds-params.py'],\n",
    "                                   sampleCols=pars.fullReportSampleCols, paramCols=pars.fullReportParamCols,\n",
    "                                   resultCols=pars.fullReportResultCols, synthCols=pars.fullReportSynthCols,\n",
    "                                   sortCols=pars.fullReportSortCols, sortAscend=pars.fullReportSortAscend,\n",
    "                                   tgtFolder=workDir,\n",
    "                                   tgtPrefix=f'{pars.studyName}{pars.subStudyName}'\n",
    "                                             f'-optanalyses{studyVariant}-report',\n",
    "                                   **pars.fullReportPlotParams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Generate the workbook report\n",
    "xlsxRep = report.toExcel(rebuild=pars.fullReportRebuild)\n",
    "\n",
    "backup(xlsxRep)  # Just in case you wanna keep reports from multiple tries with diffferent analysis specs or samples ...\n",
    "\n",
    "logger.info('Full Excel workbook report: ' + pl.Path(xlsxRep).as_posix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Generate the full HTML report\n",
    "# Note: Parallelism works well here !\n",
    "# (the default value for the \"generators\" parameter is something like the actual number of (hyper-treaded) cores of your CPU)\n",
    "htmlRep = report.toHtml(rebuild=pars.fullReportRebuild)\n",
    "\n",
    "backup(htmlRep)  # Just in case you wanna keep reports from multiple tries with diffferent analysis specs or samples ...\n",
    "                 # But warning here: it's HTML, and linked files in analysis-specific subfolders are not backup !\n",
    "\n",
    "logger.info('Full HTML report: ' + pl.Path(htmlRep).as_posix())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: Performances figures on a 6-core HT i7-10850H Ruindows 10 (2023) laptop with PCI-e SSD, \"optimal performance power scheme\", Python 3.8.15: **between 2 and 5 analysis per second with 12 parallel generators** (default number of generators = as many as hyper-threaded cores in the CPU)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix. Sumup and stats for individualised sightings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f'{pars.studyName}{pars.subStudyName}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Transects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTransects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of observers\n",
    "dfTransects.Observateur.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of field inventories = (point) transects per observer and seasonal passing\n",
    "dfTransects[['Observateur', 'Passage', 'Point']].groupby(['Observateur', 'Passage']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Total number of inventories per seasonal passing\n",
    "dfTransects.Passage.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Individualised sightings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfIndDistObs2 = dfIndDistObs.copy()\n",
    "\n",
    "dfIndDistObs2['Durée'].replace('5mn', '05mn', inplace=True)\n",
    "\n",
    "dfIndDistObs2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Total number of species\n",
    "display(dfIndDistObs2['Espèce'].nunique())\n",
    "\n",
    "# Number of species per passing and transect duration\n",
    "df = dfIndDistObs2[['Passage', 'Durée', 'Espèce']].groupby(['Passage', 'Durée']).nunique().unstack(-2).unstack(-1).to_frame().T\n",
    "df.columns = df.columns.droplevel(0)\n",
    "for p in ['a', 'b']:\n",
    "    df[(p, '10mn/05mn')] = (df[(p, '10mn')] - df[(p, '05mn')]) / df[(p, '05mn')]\n",
    "df.sort_index(axis='columns', inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nb of individuals observed per species, passing and transect duration\n",
    "df = dfIndDistObs2[['Passage', 'Durée', 'Espèce', 'Distance']].copy()\n",
    "df['Adult'] = adulte = 'All adults'\n",
    "df['Method'] = pars.subStudyName[1:]\n",
    "\n",
    "df = df.groupby(['Espèce', 'Method', 'Adult', 'Passage', 'Durée']).count().unstack(-4).unstack(-3).unstack(-2).unstack(-1)\n",
    "df.columns = df.columns.droplevel(0)\n",
    "df.fillna(0, inplace=True)\n",
    "for duree in ['05mn', '10mn']:\n",
    "    df[(pars.subStudyName[1:], adulte, 'b+a', duree)] = \\\n",
    "        df[(pars.subStudyName[1:], adulte, 'a', duree)] + df[(pars.subStudyName[1:], adulte, 'b', duree)]\n",
    "\n",
    "df.sort_values(by=[(pars.subStudyName[1:], adulte, 'b+a', '10mn')], ascending=False, inplace=True)\n",
    "df.sort_index(axis='columns', inplace=True)\n",
    "df1 = df\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nb of males observed per species, passing and transect duration\n",
    "df = dfIndDistObs2.loc[dfIndDistObs2.Adulte == 'm', ['Passage', 'Durée', 'Espèce', 'Distance']].copy()\n",
    "df['Adult'] = adulte = 'Only mâles'\n",
    "df['Method'] = pars.subStudyName[1:]\n",
    "\n",
    "df = df.groupby(['Espèce', 'Method', 'Adult', 'Passage', 'Durée']).count().unstack(-4).unstack(-3).unstack(-2).unstack(-1)\n",
    "df.columns = df.columns.droplevel(0)\n",
    "\n",
    "df.fillna(0, inplace=True)\n",
    "for duree in ['05mn', '10mn']:\n",
    "    df[(pars.subStudyName[1:], adulte, 'b+a', duree)] = \\\n",
    "        df[(pars.subStudyName[1:], adulte, 'a', duree)] + df[(pars.subStudyName[1:], adulte, 'b', duree)]\n",
    "\n",
    "df.sort_values(by=[(pars.subStudyName[1:], adulte, 'b+a', '10mn')], ascending=False, inplace=True)\n",
    "df.sort_index(axis='columns', inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nb of males and all adults observed per species, passing and transect duration\n",
    "df1.join(df, how='outer').sort_values(by=[(pars.subStudyName[1:], 'All adults', 'b+a', '10mn')],\n",
    "                                      ascending=False) # .to_excel(f'donnees/acdc/bilan-especes{pars.subStudyName}.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nb of individuals (all adults) observed per passing and transect duration\n",
    "df = dfIndDistObs2\n",
    "\n",
    "df = df[['Passage', 'Durée', 'Adulte']].groupby(['Passage', 'Durée']).count().unstack(-2)\n",
    "df.columns = df.columns.droplevel(0)\n",
    "df.sort_index(axis='columns', inplace=True)\n",
    "\n",
    "display(df)\n",
    "\n",
    "# How much more individuals (all adults) are observed in 10mn transects, when compared to 5mn ones (mean on all passes) \n",
    "print('05mn => 10mn : +', 100 * (df.loc['10mn'].sum() - df.loc['05mn'].sum()) / df.loc['05mn'].sum(), '%')\n",
    "\n",
    "# How much more individuals (all adults) are observed in 10mn transects, when compared to 5mn ones, per seasonal passing \n",
    "df = df.unstack(-1).to_frame().T\n",
    "for p in ['a', 'b']:\n",
    "    df[(p, '10mn/05mn')] = (df[(p, '10mn')] - df[(p, '05mn')]) / df[(p, '05mn')]\n",
    "df.sort_index(axis='columns', inplace=True)\n",
    "\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nb of males observed per passing and transect duration\n",
    "df = dfIndDistObs2[dfIndDistObs2.Adulte == 'm']\n",
    "\n",
    "df = df[['Passage', 'Durée', 'Adulte']].groupby(['Passage', 'Durée']).count().unstack(-2)\n",
    "df.columns = df.columns.droplevel(0)\n",
    "\n",
    "display(df)\n",
    "\n",
    "# How much more males are observed in 10mn transects, when compared to 5mn ones (mean on all passes) \n",
    "print('05mn => 10mn : +', 100 * (df.loc['10mn'].sum() - df.loc['05mn'].sum()) / df.loc['05mn'].sum(), '%')\n",
    "\n",
    "# How much more males are observed in 10mn transects, when compared to 5mn ones, per seasonal passing \n",
    "df = df.unstack(-1).to_frame().T\n",
    "for p in ['a', 'b']:\n",
    "    df[(p, '10mn/5mn')] = (df[(p, '10mn')] - df[(p, '05mn')]) / df[(p, '05mn')]\n",
    "df.sort_index(axis='columns', inplace=True)\n",
    "\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
