# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/handlers/helcom.ipynb.

# %% auto 0
__all__ = ['fname_in', 'fname_out_nc', 'zotero_key', 'ref_id', 'default_smp_types', 'fixes_nuclide_names', 'lut_nuclides',
           'coi_sediment', 'coi_val', 'coi_units_unc', 'lut_units', 'lut_dl', 'coi_dl', 'fixes_biota_species',
           'lut_biota', 'fixes_biota_tissues', 'lut_tissues', 'lut_biogroup_from_biota', 'fixes_sediments',
           'lut_sediments', 'sed_replace_lut', 'lut_filtered', 'fixes_lab_names', 'lut_lab', 'smp_type_lut',
           'lut_method', 'kw', 'load_data', 'RemapNuclideNameCB', 'ParseTimeCB', 'SplitSedimentValuesCB',
           'SanitizeValueCB', 'unc_rel2stan', 'NormalizeUncCB', 'RemapUnitCB', 'RemapDetectionLimitCB',
           'RemapSedimentCB', 'RemapFiltCB', 'combine_lut_columns', 'AddSampleIDCB', 'AddSalinityCB',
           'AddTemperatureCB', 'RemapSedSliceTopBottomCB', 'LookupDryWetPercentWeightCB', 'ParseCoordinates',
           'get_attrs', 'encode']

# %% ../../nbs/handlers/helcom.ipynb 6
import pandas as pd 
import numpy as np
#from functools import partial 
import fastcore.all as fc 
from pathlib import Path 
#from dataclasses import asdict
from typing import List, Dict, Callable, Tuple, Any 
from collections import OrderedDict, defaultdict
import re
from functools import partial

from marisco.utils import (
    Remapper, 
    ddmm_to_dd,
    Match, 
    get_unique_across_dfs
)

from marisco.callbacks import (
    Callback, 
    Transformer, 
    EncodeTimeCB, 
    AddSampleTypeIdColumnCB,
    AddNuclideIdColumnCB, 
    LowerStripNameCB, 
    SanitizeLonLatCB, 
    CompareDfsAndTfmCB, 
    RemapCB
)

from marisco.metadata import (
    GlobAttrsFeeder, 
    BboxCB, 
    DepthRangeCB, 
    TimeRangeCB, 
    ZoteroCB, 
    KeyValuePairCB
)

from marisco.configs import (
    nuc_lut_path, 
    nc_tpl_path, 
    cfg, 
    species_lut_path, 
    sediments_lut_path, 
    bodyparts_lut_path, 
    detection_limit_lut_path, 
    filtered_lut_path, 
    get_lut, 
    unit_lut_path,
    prepmet_lut_path,
    sampmet_lut_path,
    counmet_lut_path, 
    lab_lut_path,
    NC_VARS
)

from marisco.encoders import (
    NetCDFEncoder, 
)

from marisco.decoders import (
    nc_to_dfs,
    get_netcdf_properties, 
    get_netcdf_group_properties,
    get_netcdf_variable_properties
)
import warnings
warnings.filterwarnings('ignore')

# %% ../../nbs/handlers/helcom.ipynb 11
fname_in = '../../_data/accdb/mors/csv'
fname_out_nc = '../../_data/output/100-HELCOM-MORS-2024.nc'
zotero_key ='26VMZZ2Q' # HELCOM MORS zotero key
ref_id = 100 # HELCOM MORS reference id as defined by MARIS

# %% ../../nbs/handlers/helcom.ipynb 14
default_smp_types = {  
    'BIO': 'BIOTA', 
    'SEA': 'SEAWATER', 
    'SED': 'SEDIMENT'
}

# %% ../../nbs/handlers/helcom.ipynb 15
def load_data(src_dir: str|Path, 
              smp_types: dict = default_smp_types 
             ) -> Dict[str, pd.DataFrame]: 
    "Load HELCOM data and return the data in a dictionary of dataframes with the dictionary key as the sample type."
    src_path = Path(src_dir)
    
    def load_and_merge(file_prefix: str) -> pd.DataFrame:
        try:
            df_meas = pd.read_csv(src_path / f'{file_prefix}02.csv')
            df_smp = pd.read_csv(src_path / f'{file_prefix}01.csv')
            return pd.merge(df_meas, df_smp, on='KEY', how='left')
        except FileNotFoundError as e:
            print(f"Error loading files for {file_prefix}: {e}")
            return pd.DataFrame()  # Return an empty DataFrame if files are not found
    
    return {smp_type: load_and_merge(file_prefix) for file_prefix, smp_type in smp_types.items()}  

# %% ../../nbs/handlers/helcom.ipynb 35
fixes_nuclide_names = {
    'cs134137': 'cs134_137_tot',
    'cm243244': 'cm243_244_tot',
    'pu239240': 'pu239_240_tot',
    'pu238240': 'pu238_240_tot',
    'cs143': 'cs137',
    'cs145': 'cs137',
    'cs142': 'cs137',
    'cs141': 'cs137',
    'cs144': 'cs137',
    'k-40': 'k40',
    'cs140': 'cs137',
    'cs146': 'cs137',
    'cs139': 'cs137',
    'cs138': 'cs137'
    }

# %% ../../nbs/handlers/helcom.ipynb 39
# Create a lookup table for nuclide names
lut_nuclides = lambda df: Remapper(provider_lut_df=df,
                                   maris_lut_fn=nuc_lut_path,
                                   maris_col_id='nuclide_id',
                                   maris_col_name='nc_name',
                                   provider_col_to_match='value',
                                   provider_col_key='value',
                                   fname_cache='nuclides_helcom.pkl').generate_lookup_table(fixes=fixes_nuclide_names, 
                                                                                            as_df=False, overwrite=False)

# %% ../../nbs/handlers/helcom.ipynb 41
class RemapNuclideNameCB(Callback):
    "Remap data provider nuclide names to standardized MARIS nuclide names."
    def __init__(self, 
                 fn_lut: Callable # Function that returns the lookup table dictionary
                ):
        fc.store_attr()

    def __call__(self, tfm: Transformer):
        df_uniques = get_unique_across_dfs(tfm.dfs, col_name='NUCLIDE', as_df=True)
        #lut = {k: v.matched_maris_name for k, v in self.fn_lut(df_uniques).items()}    
        lut = {k: v.matched_id for k, v in self.fn_lut(df_uniques).items()}    
        for k in tfm.dfs.keys():
            tfm.dfs[k]['NUCLIDE'] = tfm.dfs[k]['NUCLIDE'].replace(lut)

# %% ../../nbs/handlers/helcom.ipynb 47
class ParseTimeCB(Callback):
    "Standardize time format across all dataframes."
    def __call__(self, tfm: Transformer):
        for df in tfm.dfs.values():
            self._process_dates(df)

    def _process_dates(self, df: pd.DataFrame) -> None:
        "Process and correct date and time information in the DataFrame."
        df['TIME'] = self._parse_date(df)
        self._handle_missing_dates(df)
        self._fill_missing_time(df)

    def _parse_date(self, df: pd.DataFrame) -> pd.Series:
        "Parse the DATE column if present."
        return pd.to_datetime(df['DATE'], format='%m/%d/%y %H:%M:%S', errors='coerce')

    def _handle_missing_dates(self, df: pd.DataFrame):
        "Handle cases where DAY or MONTH is 0 or missing."
        df.loc[df["DAY"] == 0, "DAY"] = 1
        df.loc[df["MONTH"] == 0, "MONTH"] = 1
        
        missing_day_month = (df["DAY"].isna()) & (df["MONTH"].isna()) & (df["YEAR"].notna())
        df.loc[missing_day_month, ["DAY", "MONTH"]] = 1

    def _fill_missing_time(self, df: pd.DataFrame) -> None:
        "Fill missing time values using YEAR, MONTH, and DAY columns."
        missing_time = df['TIME'].isna()
        df.loc[missing_time, 'TIME'] = pd.to_datetime(
            df.loc[missing_time, ['YEAR', 'MONTH', 'DAY']], 
            format='%Y%m%d', 
            errors='coerce'
        )

# %% ../../nbs/handlers/helcom.ipynb 59
coi_sediment = {
    'kg_type': {
        'VALUE': 'VALUE_Bq/kg',
        'UNCERTAINTY': 'ERROR%_kg',
        'DL': '< VALUE_Bq/kg',
        'UNIT': 3,  # Unit ID for Bq/kg
    },
    'm2_type': {
        'VALUE': 'VALUE_Bq/m²',
        'UNCERTAINTY': 'ERROR%_m²',
        'DL': '< VALUE_Bq/m²',
        'UNIT': 2,  # Unit ID for Bq/m²
    }
}

# %% ../../nbs/handlers/helcom.ipynb 61
class SplitSedimentValuesCB(Callback):
    "Separate sediment entries into distinct rows for Bq/kg and Bq/m² measurements."
    def __init__(self, 
                 coi: Dict[str, Dict[str, Any]] # Columns of interest with value, uncertainty, DL columns and units
                ):
        fc.store_attr()
        
    def __call__(self, tfm: Transformer):
        if 'SEDIMENT' not in tfm.dfs:
            return
            
        df = tfm.dfs['SEDIMENT']
        dfs_to_concat = []
        
        # For each measurement type (kg and m2)
        for measure_type, cols in self.coi.items():
            # If any of value/uncertainty/DL exists, keep the row
            has_data = (
                df[cols['VALUE']].notna() | 
                df[cols['UNCERTAINTY']].notna() | 
                df[cols['DL']].notna()
            )
            
            if has_data.any():
                df_measure = df[has_data].copy()
                
                # Copy columns to standardized names
                df_measure['_VALUE'] = df_measure[cols['VALUE']]
                df_measure['_UNCERTAINTY'] = df_measure[cols['UNCERTAINTY']]
                df_measure['_DL'] = df_measure[cols['DL']]
                df_measure['_UNIT'] = cols['UNIT']
                
                dfs_to_concat.append(df_measure)
        
        # Combine all measurement type dataframes
        if dfs_to_concat:
            tfm.dfs['SEDIMENT'] = pd.concat(dfs_to_concat, ignore_index=True)

# %% ../../nbs/handlers/helcom.ipynb 65
coi_val = {'SEAWATER' : {'VALUE': 'VALUE_Bq/m³'},
           'BIOTA':  {'VALUE': 'VALUE_Bq/kg'},
           'SEDIMENT': {'VALUE': '_VALUE'}}


# %% ../../nbs/handlers/helcom.ipynb 66
class SanitizeValueCB(Callback):
    "Sanitize measurement values by removing blanks and standardizing to use the `VALUE` column."
    def __init__(self, 
                 coi: Dict[str, Dict[str, str]] # Columns of interest. Format: {group_name: {'val': 'column_name'}}
                 ): 
        fc.store_attr()

    def __call__(self, tfm: Transformer):
        for grp, df in tfm.dfs.items():
            value_col = self.coi[grp]['VALUE']
            df.dropna(subset=[value_col], inplace=True)
            df['VALUE'] = df[value_col]

# %% ../../nbs/handlers/helcom.ipynb 71
def unc_rel2stan(
    df: pd.DataFrame, # DataFrame containing measurement and uncertainty columns
    meas_col: str, # Name of the column with measurement values
    unc_col: str # Name of the column with relative uncertainty values (percentages)
) -> pd.Series: # Series with calculated absolute uncertainties
    "Convert relative uncertainty to absolute uncertainty."
    return df.apply(lambda row: row[unc_col] * row[meas_col] / 100, axis=1)

# %% ../../nbs/handlers/helcom.ipynb 73
# Columns of interest
coi_units_unc = [('SEAWATER', 'VALUE_Bq/m³', 'ERROR%_m³'),
                 ('BIOTA', 'VALUE_Bq/kg', 'ERROR%'),
                 ('SEDIMENT', '_VALUE', '_UNCERTAINTY')]


# %% ../../nbs/handlers/helcom.ipynb 75
class NormalizeUncCB(Callback):
    "Convert from relative error ( % ) to standard uncertainty."
    def __init__(self, 
                 fn_convert_unc: Callable=unc_rel2stan, # Function converting relative uncertainty to absolute uncertainty
                 coi: List[Tuple[str, str, str]]=coi_units_unc # List of columns of interest
                ):
        fc.store_attr()
    
    def __call__(self, tfm: Transformer):
        for grp, val, unc in self.coi:
            if grp in tfm.dfs:
                df = tfm.dfs[grp]
                df['UNCERTAINTY'] = self.fn_convert_unc(df, val, unc)

# %% ../../nbs/handlers/helcom.ipynb 83
lut_units = {
    'SEAWATER': 1,  # 'Bq/m3'
    'SEDIMENT': 4,  # 'Bq/kgd' for sediment
    'BIOTA': {
        'D': 4,  # 'Bq/kgd'
        'W': 5,  # 'Bq/kgw'
        'F': 5   # 'Bq/kgw' (assumed to be 'Fresh', so set to wet)
    }
}

# %% ../../nbs/handlers/helcom.ipynb 84
class RemapUnitCB(Callback):
    "Set the `unit` id column in the DataFrames based on a lookup table."
    def __init__(self, 
                 lut_units: dict=lut_units # Dictionary containing renaming rules for different unit categories
                ):
        fc.store_attr()

    def __call__(self, tfm: Transformer):
        for grp in tfm.dfs.keys():
            if grp in ['SEAWATER', 'SEDIMENT']:
                tfm.dfs[grp]['UNIT'] = self.lut_units[grp]
            else:
                tfm.dfs[grp]['UNIT'] = tfm.dfs[grp]['BASIS'].apply(lambda x: lut_units[grp].get(x, 0))

# %% ../../nbs/handlers/helcom.ipynb 90
lut_dl = lambda: pd.read_excel(detection_limit_lut_path(), usecols=['name','id']).set_index('name').to_dict()['id']

# %% ../../nbs/handlers/helcom.ipynb 92
coi_dl = {'SEAWATER' : {'VALUE' : 'VALUE_Bq/m³',
                       'UNCERTAINTY' : 'ERROR%_m³',
                       'DL' : '< VALUE_Bq/m³'},
          'BIOTA':  {'VALUE' : 'VALUE_Bq/kg',
                     'UNCERTAINTY' : 'ERROR%',
                     'DL' : '< VALUE_Bq/kg'},
          'SEDIMENT': {
              'VALUE' : 'VALUE_Bq/kg',
              'UNCERTAINTY' : 'ERROR%_kg',
              'DL' : '< VALUE_Bq/kg'}}


# %% ../../nbs/handlers/helcom.ipynb 95
class RemapDetectionLimitCB(Callback):
    "Remap value type to MARIS format."
    
    def __init__(self, 
                 coi: dict,  # Configuration options for column names
                 fn_lut: Callable  # Function that returns a lookup table
                ):
        fc.store_attr()

    def __call__(self, tfm: Transformer):
        "Remap detection limits in the DataFrames using the lookup table."
        lut = self.fn_lut()
        
        for grp in tfm.dfs:
            df = tfm.dfs[grp]
            self._update_detection_limit(df, grp, lut)

    def _update_detection_limit(self, 
                                df: pd.DataFrame,  # The DataFrame to modify
                                grp: str,  # The group name to get the column configuration
                                lut: dict  # The lookup table dictionary
                               ) -> None:
        "Update detection limit column in the DataFrame based on lookup table and rules."
        
        # Check if the group exists in coi_dl
        if grp not in coi_dl:
            raise ValueError(f"Group '{grp}' not found in coi_dl configuration.")
        
        # Access column names from coi_dl
        value_col = coi_dl[grp]['VALUE']
        uncertainty_col = coi_dl[grp]['UNCERTAINTY']
        detection_col = coi_dl[grp]['DL']

        # Initialize detection limit column
        df['DL'] = df[detection_col]
        
        # Set detection limits based on conditions
        self._set_detection_limits(df, value_col, uncertainty_col, lut)

    def _set_detection_limits(self, df: pd.DataFrame, value_col: str, uncertainty_col: str, lut: dict) -> None:
        "Set detection limits based on value and uncertainty columns."
        
        # Condition for setting '='
        # 'DL' defaults to equal (i.e. '=') if there is a value and uncertainty and 'DL' value is not 
        # in the lookup table.
        
        condition_eq =(df[value_col].notna() & 
                       df[uncertainty_col].notna() & 
                       ~df['DL'].isin(lut.keys())
        )
        
        df.loc[condition_eq, 'DL'] = '='

        # Set 'Not Available' for unmatched detection limits
        df.loc[~df['DL'].isin(lut.keys()), 'DL'] = 'Not Available'
        
        # Perform lookup to map detection limits
        df['DL'] = df['DL'].map(lut)

# %% ../../nbs/handlers/helcom.ipynb 108
fixes_biota_species = {
    'CHARA BALTICA': 'NOT AVAILABLE', # CHARA BALTICA (RUBIN: CH HI;BA) is not listed in the biota data. 
    'CARDIUM EDULE': 'Cerastoderma edule',
    'LAMINARIA SACCHARINA': 'Saccharina latissima',
    'PSETTA MAXIMA': 'Scophthalmus maximus',
    'STIZOSTEDION LUCIOPERCA': 'Sander luciopercas'}

# %% ../../nbs/handlers/helcom.ipynb 112
lut_biota = lambda: Remapper(provider_lut_df=pd.read_csv(Path(fname_in) / 'RUBIN_NAME.csv'),
                             maris_lut_fn=species_lut_path,
                             maris_col_id='species_id',
                             maris_col_name='species',
                             provider_col_to_match='SCIENTIFIC NAME',
                             provider_col_key='RUBIN',
                             fname_cache='species_helcom.pkl'
                             ).generate_lookup_table(fixes=fixes_biota_species, as_df=False, overwrite=False)

# %% ../../nbs/handlers/helcom.ipynb 119
fixes_biota_tissues = {
    'WHOLE FISH WITHOUT HEAD AND ENTRAILS': 'Whole animal eviscerated without head',
    'ENTRAILS': 'Viscera',
    'SKIN/EPIDERMIS': 'Skin'}

# %% ../../nbs/handlers/helcom.ipynb 122
lut_tissues = lambda: Remapper(provider_lut_df=pd.read_csv('../../_data/accdb/mors/csv/TISSUE.csv'),
                               maris_lut_fn=bodyparts_lut_path,
                               maris_col_id='bodypar_id',
                               maris_col_name='bodypar',
                               provider_col_to_match='TISSUE_DESCRIPTION',
                               provider_col_key='TISSUE',
                               fname_cache='tissues_helcom.pkl'
                               ).generate_lookup_table(fixes=fixes_biota_tissues, as_df=False, overwrite=False)

# %% ../../nbs/handlers/helcom.ipynb 127
lut_biogroup_from_biota = lambda: get_lut(src_dir=species_lut_path().parent, fname=species_lut_path().name, 
                               key='species_id', value='biogroup_id')

# %% ../../nbs/handlers/helcom.ipynb 138
fixes_sediments = {
    'NO DATA': '(Not available)'
}

# %% ../../nbs/handlers/helcom.ipynb 141
class RemapSedimentCB(Callback):
    "Update sediment id based on MARIS species LUT (dbo_sedtype.xlsx)."
    
    def __init__(self, 
                 fn_lut: Callable,  # Function that returns the lookup table dictionary
                 sed_grp_name: str = 'SEDIMENT',  # The name of the sediment group
                 replace_lut: dict = None  # Dictionary for replacing SEDI values
                ):
        fc.store_attr()

    def __call__(self, tfm: Transformer):
        "Remap sediment types in the DataFrame using the lookup table and handle specific replacements."
        lut = self.fn_lut()
        
        # Fix inconsistent SEDI values
        tfm.dfs[self.sed_grp_name] = self._fix_inconsistent_sedi(tfm.dfs[self.sed_grp_name], self.replace_lut)
        
        # Get unique SEDI values
        unique_sedi = tfm.dfs[self.sed_grp_name]['SEDI'].unique()
        
        # Get sediment types for unique SEDI values
        sediment_mapping = self._get_sediment_types(unique_sedi, lut)
        
        # Replace SEDI values in the DataFrame using the mapping
        tfm.dfs[self.sed_grp_name]['SED_TYPE'] = tfm.dfs[self.sed_grp_name]['SEDI'].map(sediment_mapping)

    def _fix_inconsistent_sedi(self, df: pd.DataFrame, replace_lut: dict) -> pd.DataFrame:
        "Temporary fix for inconsistent SEDI values. Data provider to confirm and clarify."
        df['SEDI'] = df['SEDI'].replace(replace_lut)
        return df

    def _get_sediment_types(self, unique_sedi: np.ndarray, lut: dict) -> dict:
        "Get sediment types for unique SEDI values and return a mapping dictionary."
        sediment_mapping = {}
        
        for sedi_value in unique_sedi:
            match = lut.get(sedi_value, Match(0, None, None, None))
            if match.matched_id == 0:
                self._print_unmatched_sedi(sedi_value)
            sediment_mapping[sedi_value] = match.matched_id
        
        return sediment_mapping

    def _print_unmatched_sedi(self, 
                              sedi_value: int,  # The `SEDI` value from the DataFrame
                             ) -> None:
        "Print the SEDI value if the matched_id is 0 (i.e. Not available)."
        print(f"Unmatched SEDI: {sedi_value}")


# %% ../../nbs/handlers/helcom.ipynb 142
lut_sediments = lambda: Remapper(provider_lut_df=pd.read_csv(Path(fname_in) / 'SEDIMENT_TYPE.csv'),
                                 maris_lut_fn=sediments_lut_path,
                                 maris_col_id='sedtype_id',
                                 maris_col_name='sedtype',
                                 provider_col_to_match='SEDIMENT TYPE',
                                 provider_col_key='SEDI',
                                 fname_cache='sediments_helcom.pkl'
                                 ).generate_lookup_table(fixes=fixes_sediments, as_df=False, overwrite=False)

# %% ../../nbs/handlers/helcom.ipynb 144
sed_replace_lut = {
    56: -99,
    73: -99,
    np.nan: -99
}

# %% ../../nbs/handlers/helcom.ipynb 153
lut_filtered = {
    'N': 2, # No
    'n': 2, # No
    'F': 1 # Yes
}

# %% ../../nbs/handlers/helcom.ipynb 155
class RemapFiltCB(Callback):
    "Lookup FILT value in dataframe using the lookup table."
    def __init__(self,
                 lut_filtered: dict=lut_filtered, # Dictionary mapping FILT codes to their corresponding names
                ):
        fc.store_attr()

    def __call__(self, tfm):
        for df in tfm.dfs.values():
            if 'FILT' in df.columns:
                df['FILT'] = df['FILT'].map(lambda x: self.lut_filtered.get(x, 0))

# %% ../../nbs/handlers/helcom.ipynb 170
def combine_lut_columns(lut_path: Callable, combine_cols: List[str] = []):
    if lut_path:
        df_lut = pd.read_excel(lut_path()) 
        if combine_cols:
            # Combine the specified columns into a single column with space as separator
            df_lut['combined'] = df_lut[combine_cols].astype(str).agg(' '.join, axis=1)
            # Create a column name by joining column names with '_'
            combined_col_name = '_'.join(combine_cols)
            df_lut.rename(columns={'combined': combined_col_name}, inplace=True)
        return df_lut

# %% ../../nbs/handlers/helcom.ipynb 177
fixes_lab_names = {
    'STATENS STRÅLSKYDDSINSTITUT, SWEDEN': 'Swedish Radiation Safety Authority Sweden',
    'V. G. KHLOPIN RADIUM INSTITUTE, RUSSIA': 'V.G. Khlopin Radium Institute - Lab. of Environmental Radioactive Contamination Monitoring Russian Federation',
    'ENVIRONMENTAL PROTECTION AGENCY, LITHUANIA': 'Lithuanian Environmental Protection Agency Lithuania',
    }

# %% ../../nbs/handlers/helcom.ipynb 181
# Create a lookup table for laboratory names
lut_lab = lambda: Remapper(provider_lut_df=pd.read_csv(Path(fname_in) / 'LABORATORY_NAME.csv'),
                    maris_lut_fn= combine_lut_columns(lut_path=lab_lut_path, combine_cols=['lab','country']),
                    maris_col_id='lab_id',
                    maris_col_name='lab_country',
                    provider_col_to_match='LABORATORY_NAME',
                    provider_col_key='LABORATORY',
                    fname_cache='lab_helcom.pkl').generate_lookup_table(fixes=fixes_lab_names,as_df=False, overwrite=False) 

# %% ../../nbs/handlers/helcom.ipynb 191
smp_type_lut = {
    'SEAWATER': 1,
    'BIOTA': 2,
    'SEDIMENT': 3
}

# %% ../../nbs/handlers/helcom.ipynb 192
class AddSampleIDCB(Callback):
    "Remap `KEY` column to `SMP_ID` in each DataFrame."
    def __init__(self, lut_type: Dict[str, int]):
        self.lut_type = lut_type
        
    def __call__(self, tfm: Transformer):
        for grp in tfm.dfs:
            self._remap_sample_id(tfm.dfs[grp], grp)
    
    def _remap_sample_id(self, df: pd.DataFrame, grp: str):
        """
        Remaps the 'KEY' column to 'SMP_ID' using the provided lookup table.
        Sets 'SMP_ID' to -1 if 'LAB' or 'SEQUENCE' is NaN.
        
        Parameters:
            df (pd.DataFrame): The DataFrame to process.
            grp (str): The group key from the DataFrame dictionary, used to access specific LUT values.
        """
        # Check for NaNs in 'LAB' or 'SEQUENCE' and compute 'SMP_ID' conditionally
        df['SMP_ID'] = np.where(
            df['LAB'].isna() | df['SEQUENCE'].isna(),
            -1,
            str(self.lut_type[grp]) + df['LAB'].astype(str).str.zfill(3) + df['SEQUENCE'].astype(str).str.zfill(7)
        )

        # Convert 'SMP_ID' to integer, handling floating point representations
        df['SMP_ID'] = df['SMP_ID'].apply(lambda x: int(float(x)) if isinstance(x, str) and '.' in x else int(x))

# %% ../../nbs/handlers/helcom.ipynb 202
class AddSalinityCB(Callback):
    def __init__(self, salinity_col: str = 'SALIN'):
        self.salinity_col = salinity_col
    "Add salinity to the SEAWATER DataFrame."
    def __call__(self, tfm: Transformer):
        for df in tfm.dfs.values():
            if self.salinity_col in df.columns:
                df['SALINITY'] = df[self.salinity_col].astype(float)


# %% ../../nbs/handlers/helcom.ipynb 208
class AddTemperatureCB(Callback):
    def __init__(self, temperature_col: str = 'TTEMP'):
        self.temperature_col = temperature_col
    "Add temperature to the SEAWATER DataFrame."
    def __call__(self, tfm: Transformer):
        for df in tfm.dfs.values():
            if self.temperature_col in df.columns:
                df['TEMPERATURE'] = df[self.temperature_col].astype(float)


# %% ../../nbs/handlers/helcom.ipynb 215
lut_method = lambda: pd.read_csv(Path(fname_in) / 'ANALYSIS_METHOD.csv').set_index('METHOD').to_dict()['DESCRIPTION']

# %% ../../nbs/handlers/helcom.ipynb 221
class RemapSedSliceTopBottomCB(Callback):
    "Remap Sediment slice top and bottom to MARIS format."
    def __call__(self, tfm: Transformer):
        "Iterate through all DataFrames in the transformer object and remap sediment slice top and bottom."
        tfm.dfs['SEDIMENT']['TOP'] = tfm.dfs['SEDIMENT']['UPPSLI']
        tfm.dfs['SEDIMENT']['BOTTOM'] = tfm.dfs['SEDIMENT']['LOWSLI']

# %% ../../nbs/handlers/helcom.ipynb 246
class LookupDryWetPercentWeightCB(Callback):
    "Lookup dry-wet ratio and format for MARIS."
    def __call__(self, tfm: Transformer):
        "Iterate through all DataFrames in the transformer object and apply the dry-wet ratio lookup."
        for grp in tfm.dfs.keys():
            if 'DW%' in tfm.dfs[grp].columns:
                self._apply_dry_wet_ratio(tfm.dfs[grp])
            if 'WEIGHT' in tfm.dfs[grp].columns and 'BASIS' in tfm.dfs[grp].columns:
                self._correct_basis(tfm.dfs[grp])
                self._apply_weight(tfm.dfs[grp])

    def _apply_dry_wet_ratio(self, df: pd.DataFrame) -> None:
        "Apply dry-wet ratio conversion and formatting to the given DataFrame."
        df['PERCENTWT'] = df['DW%'] / 100  # Convert percentage to fraction
        df.loc[df['PERCENTWT'] == 0, 'PERCENTWT'] = np.NaN  # Convert 0% to NaN

    def _correct_basis(self, df: pd.DataFrame) -> None:
        "Correct BASIS values. Assuming F = Fresh weight, so F = W"
        df.loc[df['BASIS'] == 'F', 'BASIS'] = 'W'

    def _apply_weight(self, df: pd.DataFrame) -> None:
        "Apply weight conversion and formatting to the given DataFrame."
        dry_condition = df['BASIS'] == 'D'
        wet_condition = df['BASIS'] == 'W'
        
        df.loc[dry_condition, 'DRYWT'] = df['WEIGHT']
        df.loc[dry_condition & df['PERCENTWT'].notna(), 'WETWT'] = df['WEIGHT'] / df['PERCENTWT']
        
        df.loc[wet_condition, 'WETWT'] = df['WEIGHT']
        df.loc[wet_condition & df['PERCENTWT'].notna(), 'DRYWT'] = df['WEIGHT'] * df['PERCENTWT']

# %% ../../nbs/handlers/helcom.ipynb 254
class ParseCoordinates(Callback):
    "Get geographical coordinates from columns expressed in degrees decimal format  or from columns in degrees/minutes decimal format where degrees decimal format is missing or zero."
    def __init__(self, 
                 fn_convert_cor: Callable # Function that converts coordinates from degree-minute to decimal degree format
                 ):
        self.fn_convert_cor = fn_convert_cor

    def __call__(self, tfm:Transformer):
        for df in tfm.dfs.values():
            self._format_coordinates(df)

    def _format_coordinates(self, df:pd.DataFrame) -> None:
        coord_cols = self._get_coord_columns(df.columns)
        
        for coord in ['lat', 'lon']:
            decimal_col, minute_col = coord_cols[f'{coord}_d'], coord_cols[f'{coord}_m']
            
            condition = df[decimal_col].isna() | (df[decimal_col] == 0)
            df[coord.upper()] = np.where(condition,
                                 df[minute_col].apply(self._safe_convert),
                                 df[decimal_col])
        
        df.dropna(subset=['LAT', 'LON'], inplace=True)

    def _get_coord_columns(self, columns) -> dict:
        return {
            'lon_d': self._find_coord_column(columns, 'LON', 'dddddd'),
            'lat_d': self._find_coord_column(columns, 'LAT', 'dddddd'),
            'lon_m': self._find_coord_column(columns, 'LON', 'ddmmmm'),
            'lat_m': self._find_coord_column(columns, 'LAT', 'ddmmmm')
        }

    def _find_coord_column(self, columns, coord_type, coord_format) -> str:
        pattern = re.compile(f'{coord_type}.*{coord_format}', re.IGNORECASE)
        matching_columns = [col for col in columns if pattern.search(col)]
        return matching_columns[0] if matching_columns else None

    def _safe_convert(self, value) -> str:
        if pd.isna(value):
            return value
        try:
            return self.fn_convert_cor(value)
        except Exception as e:
            print(f"Error converting value {value}: {e}")
            return value

# %% ../../nbs/handlers/helcom.ipynb 265
kw = ['oceanography', 'Earth Science > Oceans > Ocean Chemistry> Radionuclides',
      'Earth Science > Human Dimensions > Environmental Impacts > Nuclear Radiation Exposure',
      'Earth Science > Oceans > Ocean Chemistry > Ocean Tracers, Earth Science > Oceans > Marine Sediments',
      'Earth Science > Oceans > Ocean Chemistry, Earth Science > Oceans > Sea Ice > Isotopes',
      'Earth Science > Oceans > Water Quality > Ocean Contaminants',
      'Earth Science > Biological Classification > Animals/Vertebrates > Fish',
      'Earth Science > Biosphere > Ecosystems > Marine Ecosystems',
      'Earth Science > Biological Classification > Animals/Invertebrates > Mollusks',
      'Earth Science > Biological Classification > Animals/Invertebrates > Arthropods > Crustaceans',
      'Earth Science > Biological Classification > Plants > Macroalgae (Seaweeds)']

# %% ../../nbs/handlers/helcom.ipynb 266
def get_attrs(
    tfm: Transformer, # Transformer object
    zotero_key: str, # Zotero dataset record key
    kw: list = kw # List of keywords
    ) -> dict: # Global attributes
    "Retrieve all global attributes."
    return GlobAttrsFeeder(tfm.dfs, cbs=[
        BboxCB(),
        DepthRangeCB(),
        TimeRangeCB(),
        ZoteroCB(zotero_key, cfg=cfg()),
        KeyValuePairCB('keywords', ', '.join(kw)),
        KeyValuePairCB('publisher_postprocess_logs', ', '.join(tfm.logs))
        ])()

# %% ../../nbs/handlers/helcom.ipynb 269
def encode(
    fname_in: str, # Input file name
    fname_out_nc: str, # Output file name
    **kwargs # Additional arguments
    ) -> None:
    "Encode data to NetCDF."
    dfs = load_data(fname_in)
    tfm = Transformer(dfs, cbs=[
                            LowerStripNameCB(col_src='NUCLIDE'),
                            RemapNuclideNameCB(lut_nuclides),
                            ParseTimeCB(),
                            EncodeTimeCB(),
                            SplitSedimentValuesCB(coi_sediment),
                            SanitizeValueCB(coi_val),       
                            NormalizeUncCB(),
                            RemapUnitCB(),
                            RemapDetectionLimitCB(coi_dl, lut_dl),                           
                            RemapCB(fn_lut=lut_biota, col_remap='SPECIES', col_src='RUBIN', dest_grps='BIOTA'),
                            RemapCB(fn_lut=lut_tissues, col_remap='BODY_PART', col_src='TISSUE', dest_grps='BIOTA'),
                            RemapCB(fn_lut=lut_biogroup_from_biota, col_remap='BIO_GROUP', col_src='SPECIES', dest_grps='BIOTA'),
                            RemapSedimentCB(fn_lut=lut_sediments, replace_lut=sed_replace_lut),
                            RemapFiltCB(lut_filtered),
                            #RemapCB(fn_lut=lut_lab, col_remap='LAB', col_src='LABORATORY', dest_grps=['BIOTA','SEDIMENT','SEAWATER']),
                            #AddSampleIDCB(lut_type=smp_type_lut),
                            AddDepthCB(),
                            AddSalinityCB(),
                            AddTemperatureCB(),
                            RemapSedSliceTopBottomCB(),
                            LookupDryWetPercentWeightCB(),
                            ParseCoordinates(ddmm_to_dd),
                            SanitizeLonLatCB()
                            ])
    tfm()
    encoder = NetCDFEncoder(tfm.dfs, 
                            dest_fname=fname_out_nc, 
                            global_attrs=get_attrs(tfm, zotero_key=zotero_key, kw=kw),
                            verbose=kwargs.get('verbose', False),
                           )
    encoder.encode()
