{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "d51d4fba-ef9b-4352-b6dc-fcab214f983f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np               # linear algebra\n",
    "import pandas as pd              # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from sklearn.metrics import pairwise_distances\n",
    "from joblib import Parallel, delayed\n",
    "import seaborn as sns\n",
    "import plotly as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ff4c0a6-6b9b-4671-8254-d5905a4e59c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = \"tiny.csv\"\n",
    "output_file = f\"results_jaccard_{input_file}\"\n",
    "passeport = \"passport_fonio.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1647797b-f826-405c-9165-c101d225321e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## passeport fonio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e8f4501-6b3c-4970-a19a-07e9674c9be2",
   "metadata": {},
   "outputs": [],
   "source": [
    "pass_df= pd.read_csv(passeport, sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd2e16fb-2a03-482d-888d-2b95c981b513",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>country</th>\n",
       "      <th>species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>Q27_02_02</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>Q27_04_01</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>Q31_02_02</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Q31_04_01</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>Q32_07_02</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            ID  country           species\n",
       "242  Q27_02_02  Nigeria  Digitaria exilis\n",
       "243  Q27_04_01  Nigeria  Digitaria exilis\n",
       "244  Q31_02_02  Nigeria  Digitaria exilis\n",
       "245  Q31_04_01  Nigeria  Digitaria exilis\n",
       "246  Q32_07_02  Nigeria  Digitaria exilis"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pass_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a64512d1-a7c1-4dcb-931d-bc13dcb43f84",
   "metadata": {},
   "outputs": [],
   "source": [
    "pass_df['ID'] = pass_df['ID'].str.replace('_','')\n",
    "pass_df['ID'] = pass_df['ID'].str.replace('-','')\n",
    "pass_df['ID'] = pass_df['ID'].str.replace('NER2011','NER')\n",
    "pass_df['ID'] = pass_df['ID'].str.replace('MNHNPP','PP')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dbabd7c-34d2-4242-a7a5-5c95776685df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>country</th>\n",
       "      <th>species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>Q270202</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>Q270401</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>Q310202</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>Q310401</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>Q320702</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Digitaria exilis</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ID  country           species\n",
       "242  Q270202  Nigeria  Digitaria exilis\n",
       "243  Q270401  Nigeria  Digitaria exilis\n",
       "244  Q310202  Nigeria  Digitaria exilis\n",
       "245  Q310401  Nigeria  Digitaria exilis\n",
       "246  Q320702  Nigeria  Digitaria exilis"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pass_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40fbdd87-e1c0-493f-9d9a-4b333ee2fb9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>species</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Digitaria exilis</th>\n",
       "      <td>199</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Digitaria iburua</th>\n",
       "      <td>21</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Digitaria longiflora</th>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Digitaria ternata</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       ID  country\n",
       "species                           \n",
       "Digitaria exilis      199      199\n",
       "Digitaria iburua       21       21\n",
       "Digitaria longiflora   13       13\n",
       "Digitaria ternata      11       11\n",
       "other                   2        2"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pass_df.groupby(['species']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a622e06c-faa6-433a-8d14-59a0c90eff5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = pass_df.groupby(['species'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "83b56848-09cf-4c96-8b04-51e43c973be1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Digitaria exilis',)\n",
      "199\n",
      "('Digitaria iburua',)\n",
      "21\n",
      "('Digitaria longiflora',)\n",
      "13\n",
      "('Digitaria ternata',)\n",
      "11\n",
      "('other',)\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "for name, group in grouped:\n",
    "    print(name)\n",
    "    print(len(group[\"ID\"].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e7438434-e3e2-48b1-a9bc-acddc1eb9ebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_file = \"SPLITTED/jaccard_100x10000.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8196448c-59d0-44da-a2ac-4e6c9e1ba910",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.formula.api as smf\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4278fa07-70c4-4dba-84a1-31a872bdccfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_file</th>\n",
       "      <th>Kmers_nb</th>\n",
       "      <th>Pop</th>\n",
       "      <th>Size_pop</th>\n",
       "      <th>Jaccard_Average_Disimilarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xaa.txt</td>\n",
       "      <td>50000</td>\n",
       "      <td>D  exilis</td>\n",
       "      <td>199</td>\n",
       "      <td>0.305329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xaa.txt</td>\n",
       "      <td>50000</td>\n",
       "      <td>D  exilis</td>\n",
       "      <td>199</td>\n",
       "      <td>0.303862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>xaa.txt</td>\n",
       "      <td>50000</td>\n",
       "      <td>D  exilis</td>\n",
       "      <td>199</td>\n",
       "      <td>0.299224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>xaa.txt</td>\n",
       "      <td>50000</td>\n",
       "      <td>D  exilis</td>\n",
       "      <td>199</td>\n",
       "      <td>0.297954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>xaa.txt</td>\n",
       "      <td>50000</td>\n",
       "      <td>D  exilis</td>\n",
       "      <td>199</td>\n",
       "      <td>0.299823</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  input_file  Kmers_nb        Pop  Size_pop  Jaccard_Average_Disimilarity\n",
       "0    xaa.txt     50000  D  exilis       199                      0.305329\n",
       "1    xaa.txt     50000  D  exilis       199                      0.303862\n",
       "2    xaa.txt     50000  D  exilis       199                      0.299224\n",
       "3    xaa.txt     50000  D  exilis       199                      0.297954\n",
       "4    xaa.txt     50000  D  exilis       199                      0.299823"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_s = pd.read_csv(results_file, sep=\"\\t\", decimal='.')\n",
    "df_s['Pop'] = df_s['Pop'].str.replace('Digitaria','D ')\n",
    "df_s.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b1f62d0-0964-4e8e-95a5-25b30b7c2f35",
   "metadata": {},
   "source": [
    "### On veut étudier ici l'influence de la pop de fonio sur la disimilarity de jaccard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ec6fdebc-4559-4f21-8f4c-f8fb372c0a7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31520 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set()\n",
    "ax = sns.boxplot(x=\"Pop\", y=\"Jaccard_Average_Disimilarity\", data=df_s, color='blue')\n",
    "plt.xlabel('Pop')\n",
    "plt.ylabel('Jaccard_Disimilarity')\n",
    "#plt.title('Boîtes à moustaches')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "943e750e-b1db-46f5-a5aa-3218b07fb93a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gp = df_s.groupby(['Pop'])\n",
    "with open('tmp_stats_pop_fonio.txt', \"w\") as fd:\n",
    "    fd.write(f'Population\\tJacc_mean\\tJaccard_var\\tJaccard_var_corr\\tJaccard_std\\tJaccard_coeffvariation\\tMean-2*std\\tMean+2*std\\n')\n",
    "    for name, group in gp:\n",
    "        fd.write(\n",
    "            f\"{name[0]}\\t{group.Jaccard_Average_Disimilarity.mean()}\\t{group.Jaccard_Average_Disimilarity.var()}\\t{group.Jaccard_Average_Disimilarity.var(ddof=1)}\\t{group.Jaccard_Average_Disimilarity.std()}\\t{(group.Jaccard_Average_Disimilarity.std()/group.Jaccard_Average_Disimilarity.mean())}\\t{group.Jaccard_Average_Disimilarity.mean() - 2*(group.Jaccard_Average_Disimilarity.std())}\\t{group.Jaccard_Average_Disimilarity.mean() + 2*(group.Jaccard_Average_Disimilarity.std())}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "684941ea-fa9e-43f0-bab0-cad5b5ace59c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Population</th>\n",
       "      <th>Jacc_mean</th>\n",
       "      <th>Jaccard_var</th>\n",
       "      <th>Jaccard_var_corr</th>\n",
       "      <th>Jaccard_std</th>\n",
       "      <th>Jaccard_coeffvariation</th>\n",
       "      <th>Mean-2*std</th>\n",
       "      <th>Mean+2*std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>D  exilis</td>\n",
       "      <td>0.283280</td>\n",
       "      <td>0.000628</td>\n",
       "      <td>0.000628</td>\n",
       "      <td>0.025053</td>\n",
       "      <td>0.088440</td>\n",
       "      <td>0.233174</td>\n",
       "      <td>0.333386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>D  iburua</td>\n",
       "      <td>0.431045</td>\n",
       "      <td>0.001138</td>\n",
       "      <td>0.001138</td>\n",
       "      <td>0.033740</td>\n",
       "      <td>0.078274</td>\n",
       "      <td>0.363566</td>\n",
       "      <td>0.498524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>D  longiflora</td>\n",
       "      <td>0.378836</td>\n",
       "      <td>0.000355</td>\n",
       "      <td>0.000355</td>\n",
       "      <td>0.018839</td>\n",
       "      <td>0.049730</td>\n",
       "      <td>0.341157</td>\n",
       "      <td>0.416515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>D  ternata</td>\n",
       "      <td>0.439251</td>\n",
       "      <td>0.002042</td>\n",
       "      <td>0.002042</td>\n",
       "      <td>0.045185</td>\n",
       "      <td>0.102869</td>\n",
       "      <td>0.348881</td>\n",
       "      <td>0.529622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>other</td>\n",
       "      <td>0.938903</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>0.012257</td>\n",
       "      <td>0.013055</td>\n",
       "      <td>0.914388</td>\n",
       "      <td>0.963418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Population  Jacc_mean  Jaccard_var  Jaccard_var_corr  Jaccard_std  \\\n",
       "0      D  exilis   0.283280     0.000628          0.000628     0.025053   \n",
       "1      D  iburua   0.431045     0.001138          0.001138     0.033740   \n",
       "2  D  longiflora   0.378836     0.000355          0.000355     0.018839   \n",
       "3     D  ternata   0.439251     0.002042          0.002042     0.045185   \n",
       "4          other   0.938903     0.000150          0.000150     0.012257   \n",
       "\n",
       "   Jaccard_coeffvariation  Mean-2*std  Mean+2*std  \n",
       "0                0.088440    0.233174    0.333386  \n",
       "1                0.078274    0.363566    0.498524  \n",
       "2                0.049730    0.341157    0.416515  \n",
       "3                0.102869    0.348881    0.529622  \n",
       "4                0.013055    0.914388    0.963418  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_stats = pd.read_csv('tmp_stats_pop_fonio.txt', sep=\"\\t\", decimal='.')\n",
    "summary_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "00c6ce0e-b280-480e-9946-89b5fdafb001",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38824 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_s.boxplot(column=\"Jaccard_Average_Disimilarity\", by=\"Pop\", vert=False)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f211e975-a66e-4f84-bb62-3030c674b6f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='Jaccard_Average_Disimilarity', ylabel='Pop'>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 38148 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.violinplot(data=df_s, x=\"Jaccard_Average_Disimilarity\", y=\"Pop\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04cdd678-cbc6-4db5-a798-c8a05828886a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.12",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
