{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!-- Table des matières automatique -->\n",
    "<h1 class='tocIgnore'>Sensitivity tests (old and unmaintained)</h1>\n",
    "\n",
    "**pyaudisam**: Automation of Distance Sampling analyses with [Distance software](http://distancesampling.org/)\n",
    "\n",
    "Copyright (C) 2021 Jean-Philippe Meuret\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify it under the terms\n",
    "of the GNU General Public License as published by the Free Software Foundation,\n",
    "either version 3 of the License, or (at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n",
    "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n",
    "See the GNU General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU General Public License along with this program.\n",
    "If not, see https://www.gnu.org/licenses/.\n",
    "\n",
    "<div style=\"overflow-y: auto\">\n",
    "  <h1>Table des matières</h1>\n",
    "  <div id=\"toc\"></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "$.getScript('ipython_notebook_toc.js')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tests de sensibilité de MCDS.exe à diverses choses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import pathlib as pl\n",
    "from packaging import version\n",
    "\n",
    "import re\n",
    "\n",
    "from collections import OrderedDict as odict\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyaudisam as ads\n",
    "\n",
    "ads.runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create temporary directory if not yet done.\n",
    "tmpDir = pl.Path('tmp')\n",
    "tmpDir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activate Warnings as Exception\n",
    "#import warnings\n",
    "#warnings.filterwarnings('error')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Communs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actual / reference closeness measure : -round(log10((actual - reference) / max(abs(actual), abs(reference))), 1)\n",
    "# = Compute the orders of magnitude that separate the difference from the max. of the two values\n",
    "def closeness(sRefAct):\n",
    "    \n",
    "    x, y = sRefAct.to_list()\n",
    "    \n",
    "    # Special cases with 1 NaN, or 1 or more inf => all different\n",
    "    if np.isnan(x):\n",
    "        if not np.isnan(y):\n",
    "            return 0 # All different\n",
    "    elif np.isnan(y):\n",
    "        return 0 # All different\n",
    "    \n",
    "    if np.isinf(x) or np.isinf(y):\n",
    "        return 0 # All different\n",
    "    \n",
    "    # Normal case\n",
    "    c = abs(x - y)\n",
    "    if not np.isnan(c) and c != 0:\n",
    "        c = c / max(abs(x), abs(y))\n",
    "    \n",
    "    return round(-np.log10(c), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare multiple variants to one of them, using closeness fn above\n",
    "# Aimed at being used as the fn to groupby(...).apply(fn) (see examples below).\n",
    "def variantCloseness(dfg, indexCols, refLabelInd=0):\n",
    "    \n",
    "    # Backup and the drop \"indexing\" columns : we don't check closeness on them\n",
    "    dfgi = dfg[indexCols].copy()\n",
    "    dfg.drop(columns=dfgi.columns.to_list(), inplace=True)\n",
    "    \n",
    "    # Compute closeness of each row to the 1st one.\n",
    "    dfgd = pd.DataFrame(columns=dfg.columns)\n",
    "    refLbl = dfg.index[0] # Label of the first row.\n",
    "    for lbl in dfg.index:\n",
    "        try:\n",
    "            dfgd.loc[lbl] = dfg.loc[[refLbl, lbl]].apply(closeness)\n",
    "        except:\n",
    "            print(lbl, refLbl)\n",
    "            print(dfg.loc[[refLbl, lbl]])\n",
    "            raise\n",
    "        \n",
    "    # Restore \"indexing\" columns : done.\n",
    "    return dfgi.join(dfgd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sensibilité de MCDS.exe à l'ordre des données"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a. Construction des variantes d'analyse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Generate test cases definition code from refout results file (don't cheat : only input columns :-)\n",
    "caseIdCols = ['Species', 'Sample', 'Model', 'DataOrder']\n",
    "\n",
    "dfAnlysCases = pd.DataFrame(data=[(spec, samp, keyFn + adjSer, dataOrd) \\\n",
    "                                  for spec in ['SYLATR', 'TURMER', 'LUSMEG', 'ALAARV', 'COLPAL',\n",
    "                                               'PHYCOL', 'EMBCIT', 'EMBCIR', 'ANTTRI', 'MILCAL'] \\\n",
    "                                  for samp in ['AB-10mn-ttdec'] \\\n",
    "                                  for keyFn in ['HNo', 'Uni', 'Haz'] \\\n",
    "                                  for adjSer in ['Cos', 'Pol'] \\\n",
    "                                  for dataOrd in ['pcdc',   # Sort by point, and increasing distances\n",
    "                                                  'pcdd',   # Sort by point, and decreasing distances\n",
    "                                                  'pc',     # Sort by point, but distance order untouched\n",
    "                                                  'dc']],   # Sort by increasing distances\n",
    "                            columns=caseIdCols)\n",
    "\n",
    "dfAnlysCases['InFileName'] = \\\n",
    "    dfAnlysCases.apply(lambda sRow: 'ACDC2019-Papyrus-{}-{}-dist.txt'.format(sRow.Species, sRow.Sample),\n",
    "                       axis='columns')\n",
    "\n",
    "dfAnlysCases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all(pl.Path('refin', sCase.InFileName).exists() \\\n",
    "           for _, sCase in dfAnlysCases.iterrows()), 'Oh, oh ... Some missing file(s) !'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dfAnlysCases = dfAnlysCases[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfAnlysCases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## b. Exécution des analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decimalFields = ['Point transect*Survey effort', 'Observation*Radial distance']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analysis engine\n",
    "mcds = ads.MCDSEngine(workDir='tmp/mcds-sens',\n",
    "                      distanceUnit='Meter', areaUnit='Hectare',\n",
    "                      surveyType='Point', distanceType='Radial', clustering=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Frozen analysis parameters (a choice here)\n",
    "KEstimCriterion = 'AIC'\n",
    "KCVInterval = 95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tsStart = pd.Timestamp.now()\n",
    "print('Started at', tsStart)\n",
    "print()\n",
    "\n",
    "# Run all analyses\n",
    "miCustCols = pd.MultiIndex.from_tuples([('sample', col, 'Value') for col in caseIdCols])\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols,\n",
    "                 data=dict(en=caseIdCols, fr=['Espèce', 'Echantillon', 'Modèle', 'OrdreDonnées']))\n",
    "\n",
    "results = ads.MCDSAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                     distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                     surveyType='Point', distanceType='Radial', clustering=False)\n",
    "\n",
    "lastInFileName = ''\n",
    "for ind, sCase in dfAnlysCases.iterrows():\n",
    "    \n",
    "    prefix = '{}-{}-{}'.format(sCase.Species, sCase.Sample, sCase.DataOrder)\n",
    "    print('#{:3d} {} {}'.format(ind+1, prefix, sCase.Model), end='\\n'*2)\n",
    "    \n",
    "    # Create data set.\n",
    "    dfInData = ads.SampleDataSet.csv2df(os.path.join('refin', sCase.InFileName), decCols=decimalFields)\n",
    "    sortCols = list()\n",
    "    sortAscg = list()\n",
    "    for srt in [sCase.DataOrder[i:i+2] for i in range(0, len(sCase.DataOrder), 2)]:\n",
    "        assert srt[0] in 'pd' and srt[1] in 'cd'\n",
    "        if srt[0] == 'p':\n",
    "            sortCols.append('Point transect*Label')\n",
    "        else: # 'd'\n",
    "            sortCols.append('Observation*Radial distance')\n",
    "        sortAscg.append(srt[1] == 'c')\n",
    "    dfInData.sort_values(by=sortCols, ascending=sortAscg, inplace=True)\n",
    "    sds = ads.SampleDataSet(dfInData, decimalFields=decimalFields)\n",
    "        \n",
    "    # Run analysis\n",
    "    analysis = ads.MCDSAnalysis(engine=mcds, sampleDataSet=sds, name=prefix,\n",
    "                                estimKeyFn=sCase.Model[:3].upper(), estimAdjustFn=sCase.Model[3:].upper(),\n",
    "                                estimCriterion=KEstimCriterion, cvInterval=KCVInterval)\n",
    "    sResult = analysis.submit().getResults()\n",
    "\n",
    "    # Save results\n",
    "    sHead = pd.Series(data=[sCase[col] for col in sCase.index[:len(caseIdCols)]], index=miCustCols)\n",
    "\n",
    "    results.append(sResult, sCustomHead=sHead)\n",
    "    \n",
    "tsEnd = pd.Timestamp.now()\n",
    "print('Finished at', tsEnd, ': duration', str(tsEnd - tsStart).replace('0 days ', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Save analysis results\n",
    "dfRes = results.dfData\n",
    "\n",
    "dfRes.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## c. Comparaison des résultats à la référence\n",
    "\n",
    "(pour chaque groupe { espèce, échantillon, modèle }, la 1ère variante de tri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove useless columns for comparison\n",
    "dfRes4c = dfRes.copy()\n",
    "dfRes4c.drop(columns=[('run output', 'run time', 'Value'), ('run output', 'run folder', 'Value'),\n",
    "                      ('detection probability', 'key function type', 'Value'),\n",
    "                      ('detection probability', 'adjustment series type', 'Value')], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare data order variant results\n",
    "miGroupCols = \\\n",
    "    pd.MultiIndex.from_tuples([('sample', col, 'Value') for col in caseIdCols if col != 'DataOrder']) \\\n",
    "                 .append(pd.MultiIndex.from_tuples([('parameters', col, 'Value') \\\n",
    "                                                    for col in dfRes['parameters'].columns.get_level_values(0)]))\n",
    "indexCols = miGroupCols.to_list() + [('sample', 'DataOrder', 'Value')]\n",
    "dfRelDif = dfRes4c.groupby(miGroupCols.to_list()).apply(variantCloseness, indexCols=indexCols, refLabelInd=0)\n",
    "\n",
    "dfRelDif"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## d. Sauvegarde des résultats."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resFileName = os.path.join(mcds.workDir, 'ACDC2019-Papyrus-auto-sens-data-order-results.xlsx')\n",
    "\n",
    "with pd.ExcelWriter(resFileName) as xlsxWriter:\n",
    "\n",
    "    dfRes.to_excel(xlsxWriter, sheet_name='RawResults', index=True)\n",
    "    dfRelDif.to_excel(xlsxWriter, sheet_name='Diff2Ref', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sensibilité de MCDS.exe à l'ordre des données (bis)\n",
    "\n",
    "Construction semi-manuelle d'un exemple de taille réduite soumise à Eric Rexstadt :\n",
    "  Cf. refout/dist-order-sens-min/dist-order-sens.odt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ordre des données générées par Distance 7 pour MCDS.exe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('refout/dist-order-sens-min/cmd-win7-dist-order/data.txt', sep='\\t',\n",
    "                 names=['region', 'area', 'point', 'effort', 'distance'])\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['npoint'] = df.point.apply(lambda s: int(s.split(' ')[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changement de l'ordre : tri par point et par distances croissantes\n",
    "df.sort_values(by=['npoint', 'distance'], inplace=True)\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pl.Path('refout/dist-order-sens-min/cmd-win7-sorted-order').mkdir(exist_ok=True)\n",
    "df[['region', 'area', 'point', 'effort', 'distance']] \\\n",
    "  .to_csv('refout/dist-order-sens-min/cmd-win7-sorted-order/data.txt', sep='\\t', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ordre des données en entrée de distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Même fichier exactement que refout/dist-order-sens-min/import-data-set.txt normalement.\n",
    "df = pd.read_csv('refin/ACDC2019-Papyrus-TURMER-AB-10mn-1dec-dist.txt', sep='\\t', header=0)\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changement de l'ordre : tri par par distances alphabétiques croissantes, oui, oui (en ignorant les points)\n",
    "# But: Voir si Distance reclasse autt par point\n",
    "df.sort_values(by=['Observation*Radial distance'], inplace=True)\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('tmp/ACDC2019-Papyrus-TURMER-AB-10mn-1dec-trialpha-dist.txt', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bac à sable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
