# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/handlers/maris_dump.ipynb.

# %% auto 0
__all__ = ['kw', 'load_dump', 'load_data', 'get_zotero_key', 'get_fname', 'get_varnames_lut', 'RemapRdnNameCB', 'renaming_rules',
           'RenameColumnCB', 'DropNAColumnsCB', 'get_dl_lut', 'SanitizeDetectionLimitCB', 'ParseTimeCB',
           'ReshapeLongToWide', 'get_attrs', 'enums_xtra', 'encode']

# %% ../../nbs/handlers/maris_dump.ipynb 4
from tqdm import tqdm
from pathlib import Path
import fastcore.all as fc
import pandas as pd

from ..callbacks import (Callback, Transformer, SanitizeLonLatCB, EncodeTimeCB)
from ..metadata import (GlobAttrsFeeder, BboxCB,
                              DepthRangeCB, TimeRangeCB,
                              ZoteroCB, KeyValuePairCB)
from ..configs import lut_path, cdl_cfg, cfg, nc_tpl_path, Enums
from ..serializers import NetCDFEncoder

# %% ../../nbs/handlers/maris_dump.ipynb 9
def load_dump(fname): 
    return pd.read_csv(fname, sep='\t', encoding='ISO-8859-1')   

def load_data(df:pd.DataFrame, # MARIS global dump 
                 ref_id:int, # Reference id of interest
                 ):
    "Load specific MARIS dataset through its ref_id"
    lut = {
        'Sediment': 'sediment',
        'Seawater': 'seawater',
        'Suspended matter': 'suspended-matter',
        'Biota': 'biota'}
    dfs = {}
    for name, grp in df[df.ref_id  == ref_id].groupby('samptype'): 
        dfs[lut[name]] = grp
    return dfs

def get_zotero_key(dfs):
    return dfs[next(iter(dfs))][['zoterourl']].iloc[0].values[0].split('/')[-1]

def get_fname(dfs):
    id, name = dfs[next(iter(dfs))][['ref_id', 'displaytext']].iloc[0]
    name = name.replace(',', '').replace('.', '').replace('-', ' ').split(' ')
    return '-'.join(([str(id)] + name)) + '.nc'

# %% ../../nbs/handlers/maris_dump.ipynb 18
def get_varnames_lut():
    fname = lut_path() / 'dbo_nuclide.xlsx'
    df_nuclide = pd.read_excel(fname, usecols=['nuclide_id', 'nc_name'])
    return df_nuclide.set_index('nuclide_id').to_dict()['nc_name']

# %% ../../nbs/handlers/maris_dump.ipynb 19
class RemapRdnNameCB(Callback):
    "Remap to MARIS radionuclide names."
    def __init__(self,
                 fn_lut=get_varnames_lut):
        fc.store_attr()

    def __call__(self, tfm):
        lut = self.fn_lut()
        for k in tfm.dfs.keys():
            tfm.dfs[k]['nuclide_id'] = tfm.dfs[k]['nuclide_id'].replace(lut)

# %% ../../nbs/handlers/maris_dump.ipynb 24
# To be added: endperiod, totdepth, lab
def renaming_rules():
    vars = cdl_cfg()['vars']
    # Define column names renaming rules
    return {
        'latitude': vars['defaults']['lat']['name'],
        'longitude': vars['defaults']['lon']['name'],
        'begperiod': vars['defaults']['time']['name'],
        'sampdepth': vars['defaults']['depth']['name'],
        'uncertaint': vars['suffixes']['uncertainty']['name'],
        'unit_id': vars['suffixes']['unit']['name'],
        'detection': vars['suffixes']['detection_limit']['name'],
        'area_id': vars['defaults']['area']['name'], 
        'species_id': vars['bio']['species']['name'],
        'biogroup_id': vars['bio']['bio_group']['name'],
        'bodypar_id': vars['bio']['body_part']['name'],
        'sedtype_id': vars['sed']['sed_type']['name'],
        'volume': vars['suffixes']['volume']['name'],
        'salinity': vars['suffixes']['salinity']['name'],
        'temperatur': vars['suffixes']['temperature']['name'],
        'sampmet_id': vars['suffixes']['sampling_method']['name'],
        'prepmet_id': vars['suffixes']['preparation_method']['name'],
        'counmet_id': vars['suffixes']['counting_method']['name'],
        'activity': 'value',
        'nuclide_id': 'nuclide'
    }

# %% ../../nbs/handlers/maris_dump.ipynb 25
class RenameColumnCB(Callback):
    "Renaming variables to MARIS standard names."
    def __init__(self, renaming_rules=renaming_rules): fc.store_attr()
    def __call__(self, tfm):
        lut = renaming_rules()
        coi = lut.keys()
        for k in tfm.dfs.keys():
            # Select cols of interest
            tfm.dfs[k] = tfm.dfs[k].loc[:, coi]
            # Rename cols
            tfm.dfs[k].rename(columns=lut, inplace=True)

# %% ../../nbs/handlers/maris_dump.ipynb 28
class DropNAColumnsCB(Callback):
    "Drop variable containing only NaN or 'Not available' (id=0 in MARIS lookup tables)."
    def __init__(self, na_value=0):
        fc.store_attr()
        
    def isMarisNA(self, col): 
        return len(col.unique()) == 1 and col.iloc[0] == self.na_value
    
    def dropMarisNA(self, df):
        na_cols = [col for col in df.columns if self.isMarisNA(df[col])]
        return df.drop(labels=na_cols, axis=1)
        
    def __call__(self, tfm):
        for k in tfm.dfs.keys():
            tfm.dfs[k] = tfm.dfs[k].dropna(axis=1, how='all')
            tfm.dfs[k] = self.dropMarisNA(tfm.dfs[k])

# %% ../../nbs/handlers/maris_dump.ipynb 31
def get_dl_lut():
    fname = lut_path() / 'dbo_detectlimit.xlsx'
    df_nuclide = pd.read_excel(fname, usecols=['name', 'id'])
    return df_nuclide.set_index('name').to_dict()['id']

# %% ../../nbs/handlers/maris_dump.ipynb 33
class SanitizeDetectionLimitCB(Callback):
    "Assign Detection Limit name to its id based on MARIS nomenclature."
    def __init__(self,
                 fn_lut=get_dl_lut):
        fc.store_attr()

    def __call__(self, tfm):
        lut = self.fn_lut()
        for k in tfm.dfs.keys():
            tfm.dfs[k]['dl'] = tfm.dfs[k]['dl'].replace(lut)

# %% ../../nbs/handlers/maris_dump.ipynb 36
class ParseTimeCB(Callback):
    def __call__(self, tfm):
        for k in tfm.dfs.keys():
            tfm.dfs[k]['time'] = pd.to_datetime(tfm.dfs[k].time, format='ISO8601')

# %% ../../nbs/handlers/maris_dump.ipynb 40
class ReshapeLongToWide(Callback):
    "Convert data from long to wide with renamed columns."
    def __init__(self, columns='nuclide', values=['value']):
        fc.store_attr()
        # Retrieve all possible derived vars (e.g 'unc', 'dl', ...) from configs
        self.derived_cols = [value['name'] for value in cdl_cfg()['vars']['suffixes'].values()]
    
    def renamed_cols(self, cols):
        "Flatten columns name"
        return [inner if outer == "value" else f'{inner}_{outer}'
                if inner else outer
                for outer, inner in cols]

    def pivot(self, df):
        # Among all possible 'derived cols' select the ones present in df
        derived_coi = [col for col in self.derived_cols if col in df.columns]
        
        df.reset_index(names='sample', inplace=True)
        
        idx = list(set(df.columns) - set([self.columns] + derived_coi + self.values))
        # print('idx: ', idx)
        # print('derived_coi: ', derived_coi)
        return df.pivot_table(index=idx,
                              columns=self.columns,
                              values=self.values + derived_coi,
                              fill_value=0,
                              aggfunc=lambda x: x
                              ).reset_index()

    def __call__(self, tfm):
        for k in tfm.dfs.keys():
            tfm.dfs[k] = self.pivot(tfm.dfs[k])
            tfm.dfs[k].columns = self.renamed_cols(tfm.dfs[k].columns)
            # tfm.dfs[k].index.name = 'sample'
            tfm.dfs[k].set_index('sample', inplace=True)

# %% ../../nbs/handlers/maris_dump.ipynb 49
kw = ['oceanography', 'Earth Science > Oceans > Ocean Chemistry> Radionuclides',
      'Earth Science > Human Dimensions > Environmental Impacts > Nuclear Radiation Exposure',
      'Earth Science > Oceans > Ocean Chemistry > Ocean Tracers, Earth Science > Oceans > Marine Sediments',
      'Earth Science > Oceans > Ocean Chemistry, Earth Science > Oceans > Sea Ice > Isotopes',
      'Earth Science > Oceans > Water Quality > Ocean Contaminants',
      'Earth Science > Biological Classification > Animals/Vertebrates > Fish',
      'Earth Science > Biosphere > Ecosystems > Marine Ecosystems',
      'Earth Science > Biological Classification > Animals/Invertebrates > Mollusks',
      'Earth Science > Biological Classification > Animals/Invertebrates > Arthropods > Crustaceans',
      'Earth Science > Biological Classification > Plants > Macroalgae (Seaweeds)']

# %% ../../nbs/handlers/maris_dump.ipynb 50
def get_attrs(tfm, zotero_key='26VMZZ2Q', kw=kw):
    return GlobAttrsFeeder(tfm.dfs, cbs=[
        BboxCB(),
        DepthRangeCB(),
        TimeRangeCB(cfg()),
        ZoteroCB(zotero_key, cfg=cfg()),
        KeyValuePairCB('keywords', ', '.join(kw)),
        KeyValuePairCB('publisher_postprocess_logs', ', '.join(tfm.logs))
        ])()

# %% ../../nbs/handlers/maris_dump.ipynb 52
def enums_xtra(tfm, vars):
    "Retrieve a subset of the lengthy enum as 'species_t' for instance"
    enums = Enums(lut_src_dir=lut_path(), cdl_enums=cdl_cfg()['enums'])
    xtras = {}
    for var in vars:
        unique_vals = tfm.unique(var)
        if unique_vals.any():
            xtras[f'{var}_t'] = enums.filter(f'{var}_t', unique_vals)
    return xtras

# %% ../../nbs/handlers/maris_dump.ipynb 53
def encode(fname_in, fname_out, nc_tpl_path, **kwargs):
    df = load_dump(fname_in)
    ref_ids = kwargs.get('ref_ids', df.ref_id.unique())
    print('Encoding ...')
    for ref_id in tqdm(ref_ids, leave=False):
        dfs = load_data(df, ref_id)
        print(get_fname(dfs))
        tfm = Transformer(dfs, cbs=[
            RemapRdnNameCB(),
            RenameColumnCB(),
            DropNAColumnsCB(),
            SanitizeDetectionLimitCB(),
            ParseTimeCB(),
            ReshapeLongToWide(),
            EncodeTimeCB(cfg()),
            SanitizeLonLatCB(verbose=True)
            ])
       
        tfm()
        encoder = NetCDFEncoder(tfm.dfs, 
                                src_fname=nc_tpl_path,
                                dest_fname=Path(fname_out) / get_fname(dfs), 
                                global_attrs=get_attrs(tfm, zotero_key=get_zotero_key(dfs), kw=kw),
                                verbose=kwargs.get('verbose', False),
                                enums_xtra=enums_xtra(tfm, vars=['species', 'body_part'])
                                )
        encoder.encode()
